Merge branch 'main' into krisp-viva-vad-support

src/pipecat/adapters/services/bedrock_adapter.py

@@ -10,7 +10,7 @@ import base64
 import copy
 import json
 from dataclasses import dataclass
-from typing import Any, Dict, List, Literal, Optional, TypedDict
+from typing import Any, Dict, List, Optional, TypedDict
 
 from loguru import logger
 
@@ -209,7 +209,7 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter[AWSBedrockLLMInvocationParams]):
                     tool_result_content = [{"json": content_json}]
                 else:
                     tool_result_content = [{"text": message["content"]}]
-            except:
+            except (json.JSONDecodeError, ValueError, AttributeError):
                 tool_result_content = [{"text": message["content"]}]
 
             return {

src/pipecat/adapters/services/gemini_adapter.py

@@ -9,7 +9,7 @@
 import base64
 import json
 from dataclasses import dataclass, field
-from typing import Any, Dict, List, Optional, Tuple, TypedDict
+from typing import Any, Dict, List, Optional, TypedDict
 
 from loguru import logger
 from openai import NotGiven
@@ -255,6 +255,9 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
         # Apply thought signatures to the corresponding messages
         self._apply_thought_signatures_to_messages(thought_signature_dicts, messages)
 
+        # When thinking is enabled, merge parallel tool calls into single messages
+        messages = self._merge_parallel_tool_calls_for_thinking(thought_signature_dicts, messages)
+
         # Check if we only have function-related messages (no regular text)
         has_regular_messages = any(
             len(msg.parts) == 1
@@ -433,6 +436,103 @@ class GeminiLLMAdapter(BaseLLMAdapter[GeminiLLMInvocationParams]):
             tool_call_id_to_name_mapping=tool_call_id_to_name_mapping,
         )
 
+    def _merge_parallel_tool_calls_for_thinking(
+        self, thought_signature_dicts: List[dict], messages: List[Content]
+    ) -> List[Content]:
+        """Merge parallel tool calls into single Content objects when thinking is enabled.
+
+        Gemini expects parallel tool calls (multiple function calls made
+        simultaneously) to be in a single Content with multiple function_call
+        Parts. This method takes a list of Content messages, where parallel
+        tool calls may be split across multiple messages, and merges them into
+        single messages.
+
+        This only has an effect when thought_signatures are present (i.e., when
+        thinking is enabled). When thinking is disabled, merging doesn't matter.
+        When thinking is enabled, there is a guarantee that the first tool call
+        (and only the first) in any batch of parallel tool calls will have a
+        thought_signature. This allows us to distinguish:
+
+        - Parallel tool calls: share a single thought_signature (on the first call)
+        - Sequential tool calls: each have their own thought_signature
+
+        Algorithm: A tool call message with a thought_signature starts a new
+        parallel group. Any tool call messages after it without a
+        thought_signature get merged into that group, regardless of what
+        messages appear in between.
+
+        Args:
+            thought_signature_dicts: A list of thought signature dicts, used
+                to determine if the work of merging is necessary.
+            messages: List of Content messages to process.
+
+        Returns:
+            List of Content messages with parallel tool calls merged when
+            thought_signatures are present, otherwise unchanged.
+        """
+        if not messages:
+            return messages
+
+        # Fast-exit if no function-call-related thought signatures
+        # This is a shortcut for determining both:
+        # - whether thinking is enabled, and
+        # - whether there are function calls in the messages
+        has_function_call_signatures = any(
+            ts.get("bookmark", {}).get("function_call") for ts in thought_signature_dicts
+        )
+        if not has_function_call_signatures:
+            return messages
+
+        def is_tool_call_message(msg: Content) -> bool:
+            """Check if message contains only function_call parts."""
+            return (
+                msg.role == "model"
+                and msg.parts
+                and all(getattr(part, "function_call", None) for part in msg.parts)
+            )
+
+        def message_has_thought_signature(msg: Content) -> bool:
+            """Check if any part in the message has a thought_signature."""
+            return any(getattr(part, "thought_signature", None) for part in msg.parts)
+
+        merged_messages = []
+        i = 0
+
+        while i < len(messages):
+            current = messages[i]
+
+            # If this is a tool call message with a thought signature, start merging
+            if is_tool_call_message(current) and message_has_thought_signature(current):
+                merged_parts = list(current.parts)
+                other_messages = []
+                j = i + 1
+
+                # Scan forward, merging tool calls without signatures, collecting others
+                while j < len(messages):
+                    next_msg = messages[j]
+                    if is_tool_call_message(next_msg):
+                        if message_has_thought_signature(next_msg):
+                            # New parallel group starts, stop here
+                            break
+                        else:
+                            # Merge this call into the current group
+                            merged_parts.extend(next_msg.parts)
+                            j += 1
+                    else:
+                        # Collect non-tool-call message, keep scanning
+                        other_messages.append(next_msg)
+                        j += 1
+
+                # Output merged calls, then collected other messages
+                merged_messages.append(Content(role="model", parts=merged_parts))
+                merged_messages.extend(other_messages)
+                i = j
+            else:
+                merged_messages.append(current)
+                i += 1
+
+        return merged_messages
+
     def _apply_thought_signatures_to_messages(
         self, thought_signature_dicts: List[dict], messages: List[Content]
     ) -> None:

src/pipecat/adapters/services/open_ai_adapter.py

@@ -7,10 +7,8 @@
 """OpenAI LLM adapter for Pipecat."""
 
 import copy
-import json
 from typing import Any, Dict, List, TypedDict
 
-from openai._types import NOT_GIVEN as OPEN_AI_NOT_GIVEN
 from openai._types import NotGiven as OpenAINotGiven
 from openai.types.chat import (
     ChatCompletionMessageParam,

src/pipecat/adapters/services/open_ai_responses_adapter.py

@@ -0,0 +1,254 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""OpenAI Responses API adapter for Pipecat."""
+
+import copy
+from typing import Any, Dict, List, Optional, TypedDict
+
+from loguru import logger
+from openai._types import NotGiven as OpenAINotGiven
+from openai.types.responses import FunctionToolParam, ResponseInputItemParam
+
+from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.processors.aggregators.llm_context import (
+    LLMContext,
+    LLMContextMessage,
+    LLMSpecificMessage,
+    NotGiven,
+)
+
+
+class OpenAIResponsesLLMInvocationParams(TypedDict, total=False):
+    """Context-based parameters for invoking OpenAI Responses API."""
+
+    input: List[ResponseInputItemParam]
+    tools: List[FunctionToolParam] | OpenAINotGiven
+    instructions: str
+
+
+class OpenAIResponsesLLMAdapter(BaseLLMAdapter[OpenAIResponsesLLMInvocationParams]):
+    """OpenAI Responses API adapter for Pipecat.
+
+    Handles:
+
+    - Converting LLMContext messages to Responses API input items
+    - Converting Pipecat's standardized tools schema to Responses API function tool format
+    - Extracting and sanitizing messages from the LLM context for logging
+    """
+
+    def __init__(self):
+        """Initialize the adapter."""
+        super().__init__()
+        self._warned_system_instruction = False
+
+    @property
+    def id_for_llm_specific_messages(self) -> str:
+        """Get the identifier used in LLMSpecificMessage instances."""
+        return "openai_responses"
+
+    def get_llm_invocation_params(
+        self,
+        context: LLMContext,
+        *,
+        system_instruction: Optional[str] = None,
+    ) -> OpenAIResponsesLLMInvocationParams:
+        """Get Responses API invocation parameters from a universal LLM context.
+
+        Args:
+            context: The LLM context containing messages, tools, etc.
+            system_instruction: Optional system instruction from service settings.
+
+        Returns:
+            Dictionary of parameters for the Responses API.
+        """
+        messages = self.get_messages(context)
+        input_items = self._convert_messages_to_input(messages)
+
+        params: OpenAIResponsesLLMInvocationParams = {
+            "input": input_items,
+            "tools": self.from_standard_tools(context.tools),
+        }
+
+        if system_instruction:
+            # Compatibility: The Responses API requires at least one input
+            # message when instructions are provided. Contexts that worked with
+            # OpenAILLMService (system_instruction + empty messages) need the
+            # instructions converted to an initial developer message.
+            #
+            # NOTE: if/when we support `previous_response_id` and/or
+            # `conversation_id`, we'll need to revisit this logic, as it'll
+            # be legit to provide instructions without input items. Worth
+            # noting that OpenAI's docs suggest these parameters are primarily
+            # for development convenience rather than performance (the model
+            # still processes the full context), and come with the tradeoff
+            # of requiring OpenAI-side 30-day conversation storage, which may
+            # not be desirable for many users. But it could give folks an easy
+            # way to store/switch between conversations without needing to
+            # manage that storage themselves.
+            if not input_items:
+                params["input"] = [{"role": "developer", "content": system_instruction}]
+            else:
+                params["instructions"] = system_instruction
+
+        return params
+
+    def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[FunctionToolParam]:
+        """Convert function schemas to Responses API function tool format.
+
+        Args:
+            tools_schema: The Pipecat tools schema to convert.
+
+        Returns:
+            List of Responses API function tool definitions.
+        """
+        functions_schema = tools_schema.standard_tools
+        result = []
+        for func in functions_schema:
+            d = func.to_default_dict()
+            tool: FunctionToolParam = {
+                "type": "function",
+                "name": d["name"],
+                "parameters": d.get("parameters", {}),
+                "strict": d.get("strict", None),
+            }
+            if "description" in d:
+                tool["description"] = d["description"]
+            result.append(tool)
+        return result
+
+    def get_messages_for_logging(self, context: LLMContext) -> List[Dict[str, Any]]:
+        """Get messages from context in a format ready for logging.
+
+        Removes or truncates sensitive data like image content for safe logging.
+
+        Args:
+            context: The LLM context containing messages.
+
+        Returns:
+            List of messages in a format ready for logging.
+        """
+        msgs = []
+        for message in self.get_messages(context):
+            msg = copy.deepcopy(message)
+            if "content" in msg:
+                if isinstance(msg["content"], list):
+                    for item in msg["content"]:
+                        if item.get("type") == "image_url":
+                            if item["image_url"]["url"].startswith("data:image/"):
+                                item["image_url"]["url"] = "data:image/..."
+                        if item.get("type") == "input_audio":
+                            item["input_audio"]["data"] = "..."
+            msgs.append(msg)
+        return msgs
+
+    def _convert_messages_to_input(
+        self, messages: List[LLMContextMessage]
+    ) -> List[ResponseInputItemParam]:
+        """Convert LLMContext messages to Responses API input items.
+
+        Args:
+            messages: Messages from the LLMContext.
+
+        Returns:
+            List of Responses API input items.
+        """
+        result: List[ResponseInputItemParam] = []
+        is_first = True
+
+        for message in messages:
+            if isinstance(message, LLMSpecificMessage):
+                result.append(message.message)
+                is_first = False
+                continue
+
+            role = message.get("role")
+
+            if role == "system":
+                if is_first and not self._warned_system_instruction:
+                    logger.warning(
+                        "System messages in LLMContext are converted to 'developer' role for the "
+                        "Responses API. Consider using settings.system_instruction instead, which "
+                        "maps to the 'instructions' parameter."
+                    )
+                    self._warned_system_instruction = True
+                content = message.get("content", "")
+                if isinstance(content, list):
+                    content = self._convert_multimodal_content(content)
+                result.append({"role": "developer", "content": content})
+
+            elif role == "user":
+                content = message.get("content", "")
+                if isinstance(content, list):
+                    content = self._convert_multimodal_content(content)
+                result.append({"role": "user", "content": content})
+
+            elif role == "assistant":
+                tool_calls = message.get("tool_calls")
+                if tool_calls:
+                    for tc in tool_calls:
+                        func = tc.get("function", {})
+                        result.append(
+                            {
+                                "type": "function_call",
+                                "call_id": tc.get("id", ""),
+                                "name": func.get("name", ""),
+                                "arguments": func.get("arguments", ""),
+                            }
+                        )
+                else:
+                    content = message.get("content", "")
+                    if isinstance(content, list):
+                        content = self._convert_multimodal_content(content)
+                    result.append({"role": "assistant", "content": content})
+
+            elif role == "tool":
+                content = message.get("content", "")
+                if not isinstance(content, str):
+                    content = str(content)
+                result.append(
+                    {
+                        "type": "function_call_output",
+                        "call_id": message.get("tool_call_id", ""),
+                        "output": content,
+                    }
+                )
+
+            is_first = False
+
+        return result
+
+    def _convert_multimodal_content(self, content: list) -> list:
+        """Convert multimodal content parts to Responses API format.
+
+        Args:
+            content: List of content parts from the LLMContext message.
+
+        Returns:
+            List of content parts in Responses API format.
+        """
+        result = []
+        for part in content:
+            part_type = part.get("type")
+            if part_type == "text":
+                result.append({"type": "input_text", "text": part.get("text", "")})
+            elif part_type == "image_url":
+                image_url_obj = part.get("image_url", {})
+                result.append(
+                    {
+                        "type": "input_image",
+                        "image_url": image_url_obj.get("url", ""),
+                        "detail": image_url_obj.get("detail", "auto"),
+                    }
+                )
+            else:
+                # Pass through other types as-is. Note: "input_audio" is not
+                # yet supported by the Responses API (coming soon per OpenAI
+                # docs) but the LLMContext format already matches the expected
+                # shape, so it should work once support is enabled.
+                result.append(part)
+        return result

src/pipecat/adapters/services/perplexity_adapter.py

@@ -0,0 +1,152 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Perplexity LLM adapter for Pipecat.
+
+Perplexity's API uses an OpenAI-compatible interface but enforces stricter
+constraints on conversation history structure:
+
+1. **Strict role alternation** — Messages must alternate between "user"/"tool"
+   and "assistant" roles. Consecutive messages with the same role (e.g. two
+   "user" messages in a row) are rejected with:
+   ``"messages must be an alternating sequence of user/tool and assistant messages"``
+
+2. **No non-initial system messages** — "system" messages are only allowed at
+   the start of the conversation. A system message after a non-system message
+   causes:
+   ``"only the initial message can have the system role"``
+
+3. **Last message must be user/tool** — The final message in the conversation
+   must have role "user" or "tool". A trailing "assistant" message causes:
+   ``"the last message must have the user or tool role"``
+
+This adapter transforms the message list to satisfy all three constraints before
+the messages are sent to Perplexity's API.
+"""
+
+import copy
+from typing import List
+
+from openai.types.chat import ChatCompletionMessageParam
+
+from pipecat.adapters.services.open_ai_adapter import OpenAILLMAdapter, OpenAILLMInvocationParams
+from pipecat.processors.aggregators.llm_context import LLMContext
+
+
+class PerplexityLLMAdapter(OpenAILLMAdapter):
+    """Adapter that transforms messages to satisfy Perplexity's API constraints.
+
+    Perplexity's API is stricter than OpenAI about message structure. This
+    adapter extends ``OpenAILLMAdapter`` and applies message transformations
+    to ensure compliance with Perplexity's constraints (role alternation,
+    no non-initial system messages, last message must be user/tool).
+
+    The transformations are applied in ``get_llm_invocation_params`` after the
+    parent adapter extracts messages from the LLM context, and before
+    ``build_chat_completion_params`` prepends ``system_instruction``.
+    """
+
+    def get_llm_invocation_params(self, context: LLMContext) -> OpenAILLMInvocationParams:
+        """Get OpenAI-compatible invocation parameters with Perplexity message fixes applied.
+
+        Args:
+            context: The LLM context containing messages, tools, etc.
+
+        Returns:
+            Dictionary of parameters for Perplexity's ChatCompletion API, with
+            messages transformed to satisfy Perplexity's constraints.
+        """
+        params = super().get_llm_invocation_params(context)
+        params["messages"] = self._transform_messages(list(params["messages"]))
+        return params
+
+    def _transform_messages(
+        self, messages: List[ChatCompletionMessageParam]
+    ) -> List[ChatCompletionMessageParam]:
+        """Transform messages to satisfy Perplexity's API constraints.
+
+        Applies three transformation steps in order:
+
+        1. **Convert non-initial system messages to user** — Any system message
+           after the initial system message block is converted to role "user",
+           since Perplexity rejects system messages after a non-system message.
+
+        2. **Merge consecutive same-role messages** — After the above
+           conversions, adjacent messages with the same role are merged using
+           list-of-dicts content format. This ensures strict role alternation
+           (e.g. a converted system→user message adjacent to an existing user
+           message gets merged).
+
+        3. **Remove trailing assistant messages** — If the last message is
+           "assistant", remove it. OpenAI appears to silently ignore trailing
+           assistant messages server-side, so removing them preserves equivalent
+           behavior while satisfying Perplexity's "last message must be
+           user/tool" constraint.
+
+        Note: we intentionally do *not* convert a trailing system message to
+        "user". That would make the transformation unstable across calls —
+        Perplexity appears to have statefulness/caching within a conversation,
+        so a message that was sent as "user" in one call but becomes "system"
+        in the next (once more messages are appended) causes errors. If the
+        context consists entirely of system messages, the Perplexity API call
+        will fail, but that mistake will be caught right away.
+
+        Args:
+            messages: List of message dicts with "role" and "content" keys.
+
+        Returns:
+            Transformed list of message dicts satisfying Perplexity's constraints.
+        """
+        if not messages:
+            return messages
+
+        messages = copy.deepcopy(messages)
+
+        # Step 1: Convert non-initial system messages to "user".
+        # Perplexity allows system messages at the start, but rejects them
+        # after any non-system message.
+        in_initial_system_block = True
+        for i in range(len(messages)):
+            if messages[i].get("role") == "system":
+                if not in_initial_system_block:
+                    messages[i]["role"] = "user"
+            else:
+                in_initial_system_block = False
+
+        # Step 2: Merge consecutive same-role messages.
+        # After system→user conversions above, we may have adjacent same-role
+        # messages that violate Perplexity's strict alternation requirement.
+        # Skip consecutive system messages at the start — Perplexity allows those.
+        i = 0
+        while i < len(messages) - 1:
+            current = messages[i]
+            next_msg = messages[i + 1]
+            if current["role"] == next_msg["role"] == "system":
+                # Perplexity allows multiple initial system messages, don't merge
+                i += 1
+            elif current["role"] == next_msg["role"]:
+                # Convert string content to list-of-dicts format for merging
+                if isinstance(current.get("content"), str):
+                    current["content"] = [{"type": "text", "text": current["content"]}]
+                if isinstance(next_msg.get("content"), str):
+                    next_msg["content"] = [{"type": "text", "text": next_msg["content"]}]
+                # Merge content from next message into current
+                if isinstance(current.get("content"), list) and isinstance(
+                    next_msg.get("content"), list
+                ):
+                    current["content"].extend(next_msg["content"])
+                messages.pop(i + 1)
+            else:
+                i += 1
+
+        # Step 3: Remove trailing assistant messages.
+        # Perplexity requires the last message to be "user" or "tool".
+        # OpenAI appears to silently ignore trailing assistant messages
+        # server-side, so removing them preserves equivalent behavior.
+        while messages and messages[-1].get("role") == "assistant":
+            messages.pop()
+
+        return messages

src/pipecat/audio/filters/aic_filter.py

@@ -9,132 +9,351 @@
 This module provides an audio filter implementation using ai-coustics' AIC SDK to
 enhance audio streams in real time. It mirrors the structure of other filters like
 the Koala filter and integrates with Pipecat's input transport pipeline.
+
+Classes:
+    AICFilter: For aic-sdk (uses 'aic_sdk' module)
+    AICModelManager: Singleton manager for read-only AIC Model instances.
 """
 
-from typing import List, Optional
+import asyncio
+from pathlib import Path
+from threading import Lock
+from typing import List, Optional, Tuple
 
 import numpy as np
+from aic_sdk import (
+    Model,
+    ParameterOutOfRangeError,
+    ProcessorAsync,
+    ProcessorConfig,
+    ProcessorParameter,
+    set_sdk_id,
+)
 from loguru import logger
 
 from pipecat.audio.filters.base_audio_filter import BaseAudioFilter
+from pipecat.audio.vad.aic_vad import AICVADAnalyzer
 from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame
 
-try:
-    # AIC SDK (https://ai-coustics.github.io/aic-sdk-py/api/)
-    from aic import AICModelType, AICParameter, Model
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
-    raise Exception(f"Missing module: {e}")
+
+class AICModelManager:
+    """Singleton manager for read-only AIC Model instances with reference counting.
+
+    Caches Model instances by path or (model_id + download_dir). Multiple
+    AICFilter instances using the same model share one Model; the manager
+    acquires on first use and releases when the last reference is dropped.
+    """
+
+    _cache: dict[str, Tuple[Model, int]] = {}  # key -> (model, ref_count)
+    _lock = Lock()
+    _loading: dict[
+        str, asyncio.Task[Model]
+    ] = {}  # key -> load task (deduplicates concurrent loads)
+
+    @classmethod
+    def _increment_reference(cls, cache_key: str, entry: Tuple[Model, int]) -> Tuple[Model, str]:
+        """Increment reference count for cached entry. Caller must hold _lock."""
+        cached_model, ref_count = entry
+        cls._cache[cache_key] = (cached_model, ref_count + 1)
+        logger.debug(f"AIC model cache key={cache_key!r} ref_count={ref_count + 1}")
+        return cached_model, cache_key
+
+    @classmethod
+    def _store_new_reference(cls, cache_key: str, model: Model) -> Tuple[Model, str]:
+        """Store new model in cache with ref count 1. Caller must hold _lock."""
+        cls._cache[cache_key] = (model, 1)
+        logger.debug(f"AIC model cached key={cache_key!r} ref_count=1")
+        return model, cache_key
+
+    @classmethod
+    async def _load_model_from_file(
+        cls,
+        cache_key: str,
+        *,
+        model_path: Optional[Path] = None,
+        model_id: Optional[str] = None,
+        model_download_dir: Optional[Path] = None,
+    ) -> Model:
+        """Run the actual load (file or download). Separate to allow create_task and deduplication."""
+        if model_path is not None:
+            logger.debug(f"Loading AIC model from file: {model_path}")
+            model_path_str = str(model_path)
+
+        elif model_id is not None and model_download_dir is not None:
+            logger.debug(f"Downloading AIC model: {model_id}")
+            model_download_dir.mkdir(parents=True, exist_ok=True)
+            model_path_str = await Model.download_async(model_id, str(model_download_dir))
+            logger.debug(f"Model downloaded to: {model_path_str}")
+
+        else:
+            raise ValueError("Unexpected model_path or (model_id and model_download_dir) state.")
+
+        loop = asyncio.get_running_loop()
+        return await loop.run_in_executor(None, lambda: Model.from_file(model_path_str))
+
+    @staticmethod
+    def _get_cache_key(
+        *,
+        model_path: Optional[Path] = None,
+        model_id: Optional[str] = None,
+        model_download_dir: Optional[Path] = None,
+    ) -> str:
+        """Build a stable cache key for the model.
+
+        Args:
+            model_path: Path to a local .aicmodel file.
+            model_id: Model identifier (See https://artifacts.ai-coustics.io/ for available models).
+            model_download_dir: Directory used for downloading models.
+
+        Returns:
+            A string key unique per (path) or (model_id + download_dir).
+        """
+        if model_path is not None:
+            return f"path:{model_path.resolve()}"
+
+        if model_id is not None and model_download_dir is not None:
+            return f"id:{model_id}:{model_download_dir.resolve()}"
+
+        raise ValueError("Either model_path or (model_id and model_download_dir) must be set.")
+
+    @classmethod
+    async def acquire(
+        cls,
+        *,
+        model_path: Optional[Path] = None,
+        model_id: Optional[str] = None,
+        model_download_dir: Optional[Path] = None,
+    ) -> Tuple[Model, str]:
+        """Get or load a Model and increment its reference count.
+
+        Call this when starting a filter. Store the returned key and pass it
+        to release() when stopping the filter.
+
+        Args:
+            model_path: Path to a local .aicmodel file. If set, model_id is ignored.
+            model_id: Model identifier to download from CDN.
+            model_download_dir: Directory for downloading models. Required if
+                model_id is used.
+
+        Returns:
+            Tuple of (shared Model instance, cache key for release).
+
+        Raises:
+            ValueError: If neither model_path nor (model_id + model_download_dir)
+                is provided, or if model_id is set without model_download_dir.
+        """
+        cache_key = cls._get_cache_key(
+            model_path=model_path,
+            model_id=model_id,
+            model_download_dir=model_download_dir,
+        )
+
+        with cls._lock:
+            entry = cls._cache.get(cache_key)
+            if entry is not None:
+                return cls._increment_reference(cache_key, entry)
+
+            # Deduplicate concurrent loads for the same key
+            load_task = cls._loading.get(cache_key)
+            if load_task is None:
+                load_task = asyncio.create_task(
+                    cls._load_model_from_file(
+                        cache_key,
+                        model_path=model_path,
+                        model_id=model_id,
+                        model_download_dir=model_download_dir,
+                    )
+                )
+                cls._loading[cache_key] = load_task
+
+        try:
+            model = await load_task
+        finally:
+            with cls._lock:
+                cls._loading.pop(cache_key, None)
+
+        with cls._lock:
+            entry = cls._cache.get(cache_key)
+            if entry is not None:
+                return cls._increment_reference(cache_key, entry)
+            return cls._store_new_reference(cache_key, model)
+
+    @classmethod
+    def release(cls, key: str) -> None:
+        """Release a reference to a cached model.
+
+        Call this when stopping a filter, with the key returned from
+        get_model(). When the last reference is released, the model
+        is removed from the cache.
+
+        Args:
+            key: Cache key returned by get_model().
+        """
+        with cls._lock:
+            entry = cls._cache.get(key)
+
+            if entry is None:
+                logger.warning(f"AIC model release unknown key={key!r}")
+                return
+
+            model, ref_count = entry
+            ref_count -= 1
+
+            if ref_count <= 0:
+                del cls._cache[key]
+                logger.debug(f"AIC model evicted key={key!r}")
+            else:
+                cls._cache[key] = (model, ref_count)
+                logger.debug(f"AIC model key={key!r} ref_count={ref_count}")
 
 
 class AICFilter(BaseAudioFilter):
     """Audio filter using ai-coustics' AIC SDK for real-time enhancement.
 
     Buffers incoming audio to the model's preferred block size and processes
-    planar frames in-place using float32 samples in the linear -1..+1 range.
+    frames using float32 samples normalized to the range -1 to +1.
     """
 
     def __init__(
         self,
         *,
-        license_key: str = "",
-        model_type: AICModelType = AICModelType.QUAIL_STT,
-        enhancement_level: Optional[float] = 1.0,
-        voice_gain: Optional[float] = 1.0,
-        noise_gate_enable: Optional[bool] = True,
+        license_key: str,
+        model_id: Optional[str] = None,
+        model_path: Optional[Path] = None,
+        model_download_dir: Optional[Path] = None,
+        enhancement_level: Optional[float] = None,
     ) -> None:
         """Initialize the AIC filter.
 
         Args:
             license_key: ai-coustics license key for authentication.
-            model_type: Model variant to load.
+            model_id: Model identifier to download from CDN. Required if model_path
+                is not provided. See https://artifacts.ai-coustics.io/ for available models.
+            model_path: Optional path to a local .aicmodel file. If provided,
+                model_id is ignored and no download occurs.
+            model_download_dir: Directory for downloading models as a Path object.
+                Defaults to a cache directory in user's home folder.
             enhancement_level: Optional overall enhancement strength (0.0..1.0).
-            voice_gain: Optional linear gain applied to detected speech (0.0..4.0).
-            noise_gate_enable: Optional enable/disable noise gate (default: True).
+                If None, the model default is used.
 
-                .. deprecated:: 1.3.0
-                    The `noise_gate_enable` parameter is deprecated and no longer has any effect.
-                    It will be removed in a future version.
+        Raises:
+            ValueError: If neither model_id nor model_path is provided, or if
+                enhancement_level is out of range.
         """
+        # Set SDK ID for telemetry identification (6 = pipecat)
+        set_sdk_id(6)
+
+        if model_id is None and model_path is None:
+            raise ValueError(
+                "Either 'model_id' or 'model_path' must be provided. "
+                "See https://artifacts.ai-coustics.io/ for available models."
+            )
+
+        if enhancement_level is not None and not 0.0 <= enhancement_level <= 1.0:
+            raise ValueError("'enhancement_level' must be between 0.0 and 1.0.")
+
         self._license_key = license_key
-        self._model_type = model_type
-
+        self._model_id = model_id
+        self._model_path = model_path
+        self._model_download_dir = model_download_dir or (
+            Path.home() / ".cache" / "pipecat" / "aic-models"
+        )
         self._enhancement_level = enhancement_level
-        self._voice_gain = voice_gain
-        if noise_gate_enable is not None:
-            import warnings
+        self._bypass = False
 
-            with warnings.catch_warnings():
-                warnings.simplefilter("always")
-                warnings.warn(
-                    "Parameter `noise_gate_enable` is deprecated and no longer has any effect. "
-                    "It will be removed in a future version. Use AIC VAD instead (create_vad_analyzer()).",
-                    DeprecationWarning,
-                )
-
-        self._noise_gate_enable = noise_gate_enable
-
-        self._enabled = True
         self._sample_rate = 0
         self._aic_ready = False
         self._frames_per_block = 0
         self._audio_buffer = bytearray()
-        # Model will be created in start() since the API now requires sample_rate
-        self._aic = None
 
-    def get_vad_factory(self):
-        """Return a zero-arg factory that will create the VAD once the model exists.
+        # Audio format constants
+        self._bytes_per_sample = 2  # int16 = 2 bytes
+        self._dtype = np.int16
+        self._scale = (
+            32768.0  # 2^15, for normalizing int16 (-32768 to 32767) to float32 (-1.0 to 1.0)
+        )
+
+        # AIC SDK objects; model is shared via AICModelManager
+        self._model_cache_key: Optional[str] = None
+        self._model = None
+        self._processor = None
+        self._processor_ctx = None
+        self._vad_ctx = None
+
+        # Pre-allocated buffers (resized in start() once frames_per_block is known)
+        self._in_f32 = None
+        self._out_i16 = None
+
+    def get_vad_context(self):
+        """Return the VAD context once the processor exists.
 
         Returns:
-            A zero-argument callable that, when invoked, returns an initialized
-            VoiceActivityDetector bound to the underlying AIC model. Raises a
-            RuntimeError if the model has not been initialized (i.e. start()
-            has not been called successfully).
+            The VadContext instance bound to the underlying processor.
+            Raises RuntimeError if the processor has not been initialized.
         """
-
-        def _factory():
-            if self._aic is None:
-                raise RuntimeError("AIC model not initialized yet. Call start(sample_rate) first.")
-            return self._aic.create_vad()
-
-        return _factory
+        if self._vad_ctx is None:
+            raise RuntimeError("AIC processor not initialized yet. Call start(sample_rate) first.")
+        return self._vad_ctx
 
     def create_vad_analyzer(
         self,
         *,
-        lookback_buffer_size: Optional[float] = None,
+        speech_hold_duration: Optional[float] = None,
+        minimum_speech_duration: Optional[float] = None,
         sensitivity: Optional[float] = None,
     ):
         """Return an analyzer that will lazily instantiate the AIC VAD when ready.
 
         AIC VAD parameters:
-          - lookback_buffer_size:
-              Number of window-length audio buffers used as a lookback buffer.
-              Higher values increase prediction stability but add latency.
-              Range: 1.0 .. 20.0, Default (SDK): 6.0
+          - speech_hold_duration:
+              How long VAD continues detecting after speech ends (in seconds).
+              Range: 0.0 to 100x model window length, Default (SDK): 0.05s
+          - minimum_speech_duration:
+              Minimum duration of speech required before VAD reports speech detected
+              (in seconds). Range: 0.0 to 1.0, Default (SDK): 0.0s
           - sensitivity:
               Energy threshold sensitivity. Energy threshold = 10 ** (-sensitivity).
-              Range: 1.0 .. 15.0, Default (SDK): 6.0
+              Range: 1.0 to 15.0, Default (SDK): 6.0
 
         Args:
-            lookback_buffer_size: Optional lookback buffer size to configure on the VAD.
-                Range: 1.0 .. 20.0. If None, SDK default is used.
+            speech_hold_duration: Optional speech hold duration to configure on the VAD.
+                If None, SDK default (0.05s) is used.
+            minimum_speech_duration: Optional minimum speech duration before VAD reports
+                speech detected. If None, SDK default (0.0s) is used.
             sensitivity: Optional sensitivity (energy threshold) to configure on the VAD.
-                Range: 1.0 .. 15.0. If None, SDK default is used.
+                Range: 1.0 to 15.0. If None, SDK default (6.0) is used.
 
         Returns:
-            A lazily-initialized AICVADAnalyzer that will bind to the VAD backend
-            once the filter's model has been created (after start(sample_rate)).
+            A lazily-initialized AICVADAnalyzer that will bind to the VAD context
+            once the filter's processor has been created (after start(sample_rate)).
         """
-        from pipecat.audio.vad.aic_vad import AICVADAnalyzer
-
         return AICVADAnalyzer(
-            vad_factory=self.get_vad_factory(),
-            lookback_buffer_size=lookback_buffer_size,
+            vad_context_factory=lambda: self.get_vad_context(),
+            speech_hold_duration=speech_hold_duration,
+            minimum_speech_duration=minimum_speech_duration,
             sensitivity=sensitivity,
         )
 
+    def _apply_enhancement_level(self):
+        """Apply enhancement_level if configured and supported by the active model."""
+        if self._processor_ctx is None or self._enhancement_level is None:
+            return
+
+        try:
+            self._processor_ctx.set_parameter(
+                ProcessorParameter.EnhancementLevel, self._enhancement_level
+            )
+        except ParameterOutOfRangeError as e:
+            logger.warning(f"AIC EnhancementLevel set_parameter out-of-range: {e}")
+            self._enhancement_level = None
+
+    def _apply_bypass(self):
+        """Apply bypass parameter to the active processor."""
+        if self._processor_ctx is None:
+            return
+
+        self._processor_ctx.set_parameter(ProcessorParameter.Bypass, 1.0 if self._bypass else 0.0)
+
     async def start(self, sample_rate: int):
         """Initialize the filter with the transport's sample rate.
 
@@ -146,55 +365,87 @@ class AICFilter(BaseAudioFilter):
         """
         self._sample_rate = sample_rate
 
+        # Acquire shared read-only model from singleton manager
+        self._model, self._model_cache_key = await AICModelManager.acquire(
+            model_path=self._model_path,
+            model_id=self._model_id,
+            model_download_dir=self._model_download_dir,
+        )
+
+        # Get optimal frames for this sample rate
+        self._frames_per_block = self._model.get_optimal_num_frames(self._sample_rate)
+
+        # Allocate processing buffers now that we know the block size
+        self._in_f32 = np.zeros((1, self._frames_per_block), dtype=np.float32)
+        self._out_i16 = np.zeros(self._frames_per_block, dtype=np.int16)
+
+        # Create configuration
+        config = ProcessorConfig.optimal(
+            self._model,
+            sample_rate=self._sample_rate,
+        )
+
+        # Create async processor
         try:
-            # Create model with required runtime parameters
-            self._aic = Model(
-                model_type=self._model_type,
-                license_key=self._license_key or None,
-                sample_rate=self._sample_rate,
-                channels=1,
-            )
-            self._frames_per_block = self._aic.optimal_num_frames()
-
-            # Optional parameter configuration
-            if self._enhancement_level is not None:
-                self._aic.set_parameter(
-                    AICParameter.ENHANCEMENT_LEVEL,
-                    float(self._enhancement_level if self._enabled else 0.0),
-                )
-            if self._voice_gain is not None:
-                self._aic.set_parameter(AICParameter.VOICE_GAIN, float(self._voice_gain))
-
-            self._aic_ready = True
-
-            # Log processor information
-            logger.debug(f"ai-coustics filter started:")
-            logger.debug(f"  Sample rate: {self._sample_rate} Hz")
-            logger.debug(f"  Frames per chunk: {self._frames_per_block}")
-            logger.debug(f"  Enhancement strength: {int(self._enhancement_level * 100)}%")
-            logger.debug(f"  Optimal input buffer size: {self._aic.optimal_num_frames()} samples")
-            logger.debug(f"  Optimal sample rate: {self._aic.optimal_sample_rate()} Hz")
-            logger.debug(
-                f"  Current algorithmic latency: {self._aic.processing_latency() / self._sample_rate * 1000:.2f}ms"
-            )
+            self._processor = ProcessorAsync(self._model, self._license_key, config)
         except Exception as e:  # noqa: BLE001 - surfacing SDK initialization errors
             logger.error(f"AIC model initialization failed: {e}")
-            self._aic_ready = False
+            self._processor = None
+
+        self._aic_ready = self._processor is not None
+
+        if not self._aic_ready:
+            logger.debug(f"ai-coustics filter is not ready.")
+            return
+
+        # Get contexts for parameter control and VAD
+        self._processor_ctx = self._processor.get_processor_context()
+        self._vad_ctx = self._processor.get_vad_context()
+
+        # Apply initial control parameters
+        self._apply_bypass()
+        self._apply_enhancement_level()
+
+        # Log processor information
+        logger.debug(f"ai-coustics filter started:")
+        logger.debug(f"  Model ID: {self._model.get_id()}")
+        logger.debug(f"  Sample rate: {self._sample_rate} Hz")
+        logger.debug(f"  Frames per chunk: {self._frames_per_block}")
+        if self._enhancement_level is not None:
+            logger.debug(f"  Enhancement level: {self._enhancement_level}")
+        else:
+            logger.debug("  Enhancement level not configured; using the model's default behavior.")
+        logger.debug(f"  Optimal sample rate: {self._model.get_optimal_sample_rate()} Hz")
+        logger.debug(
+            f"  Optimal number of frames for {self._sample_rate} Hz: "
+            f"{self._model.get_optimal_num_frames(self._sample_rate)}"
+        )
+        logger.debug(
+            f"  Output delay: {self._processor_ctx.get_output_delay()} samples "
+            f"({self._processor_ctx.get_output_delay() / self._sample_rate * 1000:.2f}ms)"
+        )
 
     async def stop(self):
-        """Clean up the AIC model when stopping.
+        """Clean up the AIC processor when stopping.
 
         Returns:
             None
         """
         try:
-            if self._aic is not None:
-                self._aic.close()
+            if self._processor_ctx is not None:
+                self._processor_ctx.reset()
         finally:
-            self._aic = None
+            self._processor = None
+            self._processor_ctx = None
+            self._vad_ctx = None
+            self._model = None
             self._aic_ready = False
             self._audio_buffer.clear()
 
+            if self._model_cache_key is not None:
+                AICModelManager.release(self._model_cache_key)
+                self._model_cache_key = None
+
     async def process_frame(self, frame: FilterControlFrame):
         """Process control frames to enable/disable filtering.
 
@@ -205,11 +456,11 @@ class AICFilter(BaseAudioFilter):
             None
         """
         if isinstance(frame, FilterEnableFrame):
-            self._enabled = frame.enable
-            if self._aic is not None:
+            self._bypass = not frame.enable
+            if self._processor_ctx is not None:
                 try:
-                    level = float(self._enhancement_level if self._enabled else 0.0)
-                    self._aic.set_parameter(AICParameter.ENHANCEMENT_LEVEL, level)
+                    self._apply_bypass()
+                    self._apply_enhancement_level()
                 except Exception as e:  # noqa: BLE001
                     logger.error(f"AIC set_parameter failed: {e}")
 
@@ -220,43 +471,43 @@ class AICFilter(BaseAudioFilter):
         model's required block length. Returns enhanced audio data.
 
         Args:
-            audio: Raw audio data as bytes to be filtered (int16 PCM, planar).
+            audio: Raw audio data as bytes (int16 PCM).
 
         Returns:
-            Enhanced audio data as bytes (int16 PCM, planar).
+            Enhanced audio data as bytes (int16 PCM).
         """
-        if not self._aic_ready or self._aic is None:
+        if not self._aic_ready or self._processor is None:
             return audio
 
         self._audio_buffer.extend(audio)
+        available_frames = len(self._audio_buffer) // self._bytes_per_sample
+        num_blocks = available_frames // self._frames_per_block
+
+        if num_blocks == 0:
+            return b""
+
+        block_size = self._frames_per_block * self._bytes_per_sample
+        total_size = num_blocks * block_size
+        blocks_data = bytes(self._audio_buffer[:total_size])
+        self._audio_buffer = self._audio_buffer[total_size:]
 
         filtered_chunks: List[bytes] = []
 
-        # Number of int16 samples currently buffered
-        available_frames = len(self._audio_buffer) // 2
+        for i in range(num_blocks):
+            start = i * block_size
+            block_i16 = np.frombuffer(blocks_data[start : start + block_size], dtype=self._dtype)
 
-        while available_frames >= self._frames_per_block:
-            # Consume exactly one block worth of frames
-            samples_to_consume = self._frames_per_block * 1
-            bytes_to_consume = samples_to_consume * 2
-            block_bytes = bytes(self._audio_buffer[:bytes_to_consume])
+            # Reuse input buffer, in-place divide
+            np.copyto(self._in_f32[0], block_i16)
+            self._in_f32 /= self._scale
 
-            # Convert to float32 in -1..+1 range and reshape to planar (channels, frames)
-            block_i16 = np.frombuffer(block_bytes, dtype=np.int16)
-            block_f32 = (block_i16.astype(np.float32) / 32768.0).reshape(
-                (1, self._frames_per_block)
-            )
+            out_f32 = await self._processor.process_async(self._in_f32)
 
-            # Process planar in-place; returns ndarray (same shape)
-            out_f32 = await self._aic.process_async(block_f32)
+            # Convert float32 output back to int16
+            np.multiply(out_f32, self._scale, out=self._in_f32)  # reuse in_f32 as temp
+            np.clip(self._in_f32, -self._scale, self._scale - 1, out=self._in_f32)
+            np.copyto(self._out_i16, self._in_f32[0].astype(self._dtype))
 
-            # Convert back to int16 bytes, planar layout
-            out_i16 = np.clip(out_f32 * 32768.0, -32768, 32767).astype(np.int16)
-            filtered_chunks.append(out_i16.reshape(-1).tobytes())
+            filtered_chunks.append(self._out_i16.tobytes())
 
-            # Slide buffer
-            self._audio_buffer = self._audio_buffer[bytes_to_consume:]
-            available_frames = len(self._audio_buffer) // 2
-
-        # Do not flush incomplete frames; keep them buffered for the next call
         return b"".join(filtered_chunks)

src/pipecat/audio/filters/krisp_filter.py

@@ -61,6 +61,7 @@ class KrispFilter(BaseAudioFilter):
     Provides real-time noise reduction for audio streams using Krisp's
     proprietary noise suppression algorithms. Requires a Krisp model file
     for operation.
+
     .. deprecated:: 0.0.94
         The KrispFilter is deprecated and will be removed in a future version.
         Use KrispVivaFilter instead.

src/pipecat/audio/filters/krisp_viva_filter.py

@@ -9,7 +9,6 @@
 This module provides an audio filter implementation using Krisp VIVA SDK.
 """
 
-import asyncio
 import os
 
 import numpy as np
@@ -40,7 +39,11 @@ class KrispVivaFilter(BaseAudioFilter):
     """
 
     def __init__(
-        self, model_path: str = None, frame_duration: int = 10, noise_suppression_level: int = 100
+        self,
+        model_path: str = None,
+        frame_duration: int = 10,
+        noise_suppression_level: int = 100,
+        api_key: str = "",
     ) -> None:
         """Initialize the Krisp noise reduction filter.
 
@@ -49,6 +52,8 @@ class KrispVivaFilter(BaseAudioFilter):
                 If None, uses KRISP_VIVA_FILTER_MODEL_PATH environment variable.
             frame_duration: Frame duration in milliseconds.
             noise_suppression_level: Noise suppression level.
+            api_key: Krisp SDK API key. If empty, falls back to
+                the KRISP_VIVA_API_KEY environment variable.
 
         Raises:
             ValueError: If model_path is not provided and KRISP_VIVA_FILTER_MODEL_PATH is not set.
@@ -58,6 +63,8 @@ class KrispVivaFilter(BaseAudioFilter):
         """
         super().__init__()
 
+        self._api_key = api_key
+
         try:
             # Set model path, checking environment if not specified
             if model_path:
@@ -133,7 +140,7 @@ class KrispVivaFilter(BaseAudioFilter):
         """
         try:
             # Acquire SDK reference (will initialize on first call)
-            KrispVivaSDKManager.acquire()
+            KrispVivaSDKManager.acquire(api_key=self._api_key)
             self._session = self._create_session(sample_rate, self._frame_duration_ms)
         except Exception as e:
             logger.error(f"Failed to start Krisp session: {e}", exc_info=True)

src/pipecat/audio/krisp_instance.py

@@ -7,6 +7,7 @@
 """Krisp Instance manager for pipecat audio."""
 
 import atexit
+import os
 from threading import Lock
 
 from loguru import logger
@@ -88,17 +89,26 @@ class KrispVivaSDKManager:
     _lock = Lock()
     _reference_count = 0
 
+    @staticmethod
+    def _license_callback(error, error_message):
+        """Callback for Krisp SDK licensing errors."""
+        logger.error(f"Krisp licensing error: {error} - {error_message}")
+
     @staticmethod
     def _log_callback(log_message, log_level):
         """Thread-safe callback for Krisp SDK logging."""
         logger.info(f"[{log_level}] {log_message}")
 
     @classmethod
-    def acquire(cls):
+    def acquire(cls, api_key: str = ""):
         """Acquire a reference to the SDK (initializes if needed).
 
         Call this when creating a filter instance.
 
+        Args:
+            api_key: Krisp SDK API key. If empty, falls back to the
+                KRISP_VIVA_API_KEY environment variable.
+
         Raises:
             Exception: If SDK initialization fails (propagated from krisp_audio)
         """
@@ -106,7 +116,19 @@ class KrispVivaSDKManager:
             # Initialize SDK on first acquire
             if cls._reference_count == 0:
                 try:
-                    krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
+                    key = api_key or os.environ.get("KRISP_VIVA_API_KEY", "")
+                    try:
+                        # New SDK signature (requires license key)
+                        krisp_audio.globalInit(
+                            "",
+                            key,
+                            cls._license_callback,
+                            cls._log_callback,
+                            krisp_audio.LogLevel.Off,
+                        )
+                    except TypeError:
+                        # Old SDK signature (no license key)
+                        krisp_audio.globalInit("", cls._log_callback, krisp_audio.LogLevel.Off)
 
                     cls._initialized = True
 

src/pipecat/audio/turn/krisp_viva_turn.py

@@ -15,6 +15,7 @@ passed directly to the constructor.
 """
 
 import os
+import time
 from typing import Optional, Tuple
 
 import numpy as np
@@ -26,7 +27,7 @@ from pipecat.audio.krisp_instance import (
     int_to_krisp_sample_rate,
 )
 from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
-from pipecat.metrics.metrics import MetricsData
+from pipecat.metrics.metrics import MetricsData, TurnMetricsData
 
 try:
     import krisp_audio
@@ -63,6 +64,7 @@ class KrispVivaTurn(BaseTurnAnalyzer):
         model_path: Optional[str] = None,
         sample_rate: Optional[int] = None,
         params: Optional[KrispTurnParams] = None,
+        api_key: str = "",
     ) -> None:
         """Initialize the Krisp turn analyzer.
 
@@ -72,6 +74,8 @@ class KrispVivaTurn(BaseTurnAnalyzer):
             sample_rate: Optional initial sample rate for audio processing.
                 If provided, this will be used as the fixed sample rate.
             params: Configuration parameters for turn analysis behavior.
+            api_key: Krisp SDK API key. If empty, falls back to
+                the KRISP_VIVA_API_KEY environment variable.
 
         Raises:
             ValueError: If model_path is not provided and KRISP_VIVA_TURN_MODEL_PATH is not set.
@@ -83,7 +87,7 @@ class KrispVivaTurn(BaseTurnAnalyzer):
 
         # Acquire SDK reference (will initialize on first call)
         try:
-            KrispVivaSDKManager.acquire()
+            KrispVivaSDKManager.acquire(api_key=api_key)
             self._sdk_acquired = True
         except Exception as e:
             self._sdk_acquired = False
@@ -115,6 +119,9 @@ class KrispVivaTurn(BaseTurnAnalyzer):
             self._last_probability = None
             self._frame_probabilities = []
             self._last_state = EndOfTurnState.INCOMPLETE
+            self._speech_stopped_time: Optional[float] = None
+            self._e2e_processing_time_ms: Optional[float] = None
+            self._last_metrics: Optional[TurnMetricsData] = None
 
             # Create session with provided sample rate or default to 16000 Hz
             # This preloads the model to improve latency when set_sample_rate is called later
@@ -288,7 +295,14 @@ class KrispVivaTurn(BaseTurnAnalyzer):
                     # Track speech start time
                     if not self._speech_triggered:
                         logger.trace("Speech detected, turn analysis started")
+                        self._e2e_processing_time_ms = None
                     self._speech_triggered = True
+                    # Reset speech stopped time when speech resumes
+                    self._speech_stopped_time = None
+                else:
+                    # Record the moment speech transitions to non-speech
+                    if self._speech_triggered and self._speech_stopped_time is None:
+                        self._speech_stopped_time = time.perf_counter()
                 # Note: We don't immediately mark as complete on silence detection.
                 # Instead, we wait for the model's probability check below to confirm
                 # end-of-turn based on the threshold.
@@ -308,6 +322,18 @@ class KrispVivaTurn(BaseTurnAnalyzer):
                 # Only mark as complete if we've detected speech and the model
                 # confirms with sufficient confidence
                 if self._speech_triggered and prob >= self._params.threshold:
+                    # Calculate e2e processing time: time from speech stop to threshold crossing
+                    if self._speech_stopped_time is not None:
+                        self._e2e_processing_time_ms = (
+                            time.perf_counter() - self._speech_stopped_time
+                        ) * 1000
+                        self._last_metrics = TurnMetricsData(
+                            processor="KrispVivaTurn",
+                            is_complete=True,
+                            probability=prob,
+                            e2e_processing_time_ms=self._e2e_processing_time_ms,
+                        )
+                        logger.debug(f"Krisp turn complete")
                     state = EndOfTurnState.COMPLETE
                     self.clear()
                     break
@@ -329,12 +355,15 @@ class KrispVivaTurn(BaseTurnAnalyzer):
             Tuple containing the end-of-turn state and optional metrics data.
             Returns the last state determined by append_audio().
         """
-        # For real-time processing, the state is determined in append_audio
-        # Return the last state that was computed
-        return self._last_state, None
+        # For real-time processing, the state is determined in append_audio.
+        # Consume metrics so they aren't pushed twice.
+        metrics = self._last_metrics
+        self._last_metrics = None
+        return self._last_state, metrics
 
     def clear(self):
         """Reset the turn analyzer to its initial state."""
         self._speech_triggered = False
         self._audio_buffer.clear()
         self._last_state = EndOfTurnState.INCOMPLETE
+        self._speech_stopped_time = None

src/pipecat/audio/turn/smart_turn/base_smart_turn.py

@@ -21,7 +21,7 @@ import numpy as np
 from loguru import logger
 
 from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
-from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData
+from pipecat.metrics.metrics import MetricsData, TurnMetricsData
 
 # Default timing parameters
 STOP_SECS = 3
@@ -222,18 +222,11 @@ class BaseSmartTurn(BaseTurnAnalyzer):
                 # Calculate processing time
                 e2e_processing_time_ms = (end_time - start_time) * 1000
 
-                # Extract metrics from the nested structure
-                metrics = result.get("metrics", {})
-                inference_time = metrics.get("inference_time", 0)
-                total_time = metrics.get("total_time", 0)
-
                 # Prepare the result data
-                result_data = SmartTurnMetricsData(
+                result_data = TurnMetricsData(
                     processor="BaseSmartTurn",
                     is_complete=result["prediction"] == 1,
                     probability=result["probability"],
-                    inference_time_ms=inference_time * 1000,
-                    server_total_time_ms=total_time * 1000,
                     e2e_processing_time_ms=e2e_processing_time_ms,
                 )
 
@@ -241,8 +234,6 @@ class BaseSmartTurn(BaseTurnAnalyzer):
                     f"Prediction: {'Complete' if result_data.is_complete else 'Incomplete'}"
                 )
                 logger.trace(f"Probability of complete: {result_data.probability:.4f}")
-                logger.trace(f"Inference time: {result_data.inference_time_ms:.2f}ms")
-                logger.trace(f"Server total time: {result_data.server_total_time_ms:.2f}ms")
                 logger.trace(f"E2E processing time: {result_data.e2e_processing_time_ms:.2f}ms")
             except SmartTurnTimeoutException:
                 logger.debug(

src/pipecat/audio/turn/smart_turn/local_coreml_smart_turn.py

@@ -10,6 +10,7 @@ This module provides a smart turn analyzer that uses CoreML models for
 local end-of-turn detection without requiring network connectivity.
 """
 
+import warnings
 from typing import Any, Dict
 
 import numpy as np
@@ -35,6 +36,10 @@ class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
     Provides end-of-turn detection using locally-stored CoreML models,
     enabling offline operation without network dependencies. Optimized
     for Apple Silicon and other CoreML-compatible hardware.
+
+    .. deprecated:: 0.0.106
+        LocalCoreMLSmartTurnAnalyzer is deprecated and will be removed in a future version.
+        Use LocalSmartTurnAnalyzerV3 instead.
     """
 
     def __init__(self, *, smart_turn_model_path: str, **kwargs):
@@ -50,6 +55,15 @@ class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
         """
         super().__init__(**kwargs)
 
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "LocalCoreMLSmartTurnAnalyzer is deprecated and will be removed in a future "
+                "version. Use LocalSmartTurnAnalyzerV3 instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         if not smart_turn_model_path:
             logger.error("smart_turn_model_path is not set.")
             raise Exception("smart_turn_model_path must be provided.")

src/pipecat/audio/turn/smart_turn/local_smart_turn.py

@@ -36,7 +36,7 @@ class LocalSmartTurnAnalyzer(BaseSmartTurn):
     enabling offline operation without network dependencies. Uses
     Wav2Vec2-BERT architecture for audio sequence classification.
 
-    .. deprecated:: 0.98.0
+    .. deprecated:: 0.0.98
         LocalSmartTurnAnalyzer is deprecated and will be removed in a future version.
         Use LocalSmartTurnAnalyzerV3 instead.
     """

src/pipecat/audio/turn/smart_turn/local_smart_turn_v2.py

@@ -10,6 +10,7 @@ This module provides a smart turn analyzer that uses PyTorch models for
 local end-of-turn detection without requiring network connectivity.
 """
 
+import warnings
 from typing import Any, Dict
 
 import numpy as np
@@ -41,6 +42,10 @@ class LocalSmartTurnAnalyzerV2(BaseSmartTurn):
     Provides end-of-turn detection using locally-stored PyTorch models,
     enabling offline operation without network dependencies. Uses
     Wav2Vec2 architecture for audio sequence classification.
+
+    .. deprecated:: 0.0.106
+        LocalSmartTurnAnalyzerV2 is deprecated and will be removed in a future version.
+        Use LocalSmartTurnAnalyzerV3 instead.
     """
 
     def __init__(self, *, smart_turn_model_path: str, **kwargs):
@@ -53,6 +58,15 @@ class LocalSmartTurnAnalyzerV2(BaseSmartTurn):
         """
         super().__init__(**kwargs)
 
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "LocalSmartTurnAnalyzerV2 is deprecated and will be removed in a future version. "
+                "Use LocalSmartTurnAnalyzerV3 instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
         if not smart_turn_model_path:
             # Define the path to the pretrained model on Hugging Face
             smart_turn_model_path = "pipecat-ai/smart-turn-v2"

src/pipecat/audio/turn/smart_turn/local_smart_turn_v3.py

@@ -13,19 +13,16 @@ local end-of-turn detection without requiring network connectivity.
 from typing import Any, Dict, Optional
 
 import numpy as np
+import onnxruntime as ort
+import soxr
 from loguru import logger
+from transformers import WhisperFeatureExtractor
 
 from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn
+from pipecat.utils.env import env_truthy
 
-try:
-    import onnxruntime as ort
-    from transformers import WhisperFeatureExtractor
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use LocalSmartTurnAnalyzerV3, you need to `pip install pipecat-ai[local-smart-turn-v3]`."
-    )
-    raise Exception(f"Missing module: {e}")
+# The Whisper-based ONNX model expects 16 kHz audio input.
+_MODEL_SAMPLE_RATE = 16000
 
 
 class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
@@ -48,6 +45,8 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
         """
         super().__init__(**kwargs)
 
+        self._log_data = env_truthy("PIPECAT_SMART_TURN_LOG_DATA", default=False)
+
         if not smart_turn_model_path:
             # Load bundled model
             model_name = "smart-turn-v3.2-cpu.onnx"
@@ -81,10 +80,70 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
 
         logger.debug("Loaded Local Smart Turn v3.x")
 
+    def _write_audio_to_wav(
+        self, audio_array: np.ndarray, sample_rate: int = _MODEL_SAMPLE_RATE, suffix: str = ""
+    ) -> None:
+        """Write audio data to a WAV file in a background thread.
+
+        Args:
+            audio_array: The audio data as a numpy array (float32, normalized to [-1, 1]).
+            sample_rate: The sample rate of the audio data.
+            suffix: Optional suffix to append to the filename (e.g., "_raw", "_padded").
+        """
+        import os
+        import threading
+        import wave
+        from datetime import datetime
+
+        # Generate filename with current timestamp (millisecond precision)
+        timestamp = datetime.now().strftime("%Y-%m-%d__%H:%M:%S.%f")[:-3]
+        log_dir = "./smart_turn_audio_log"
+        os.makedirs(log_dir, exist_ok=True)
+        filename = os.path.join(log_dir, f"{timestamp}{suffix}.wav")
+
+        # Make a copy of the audio data to avoid issues with the array being modified
+        audio_copy = audio_array.copy()
+
+        def write_wav():
+            try:
+                # Convert float32 audio to int16 for WAV file
+                audio_int16 = (audio_copy * 32767).astype(np.int16)
+
+                with wave.open(filename, "wb") as wav_file:
+                    wav_file.setnchannels(1)  # Mono
+                    wav_file.setsampwidth(2)  # 2 bytes for int16
+                    wav_file.setframerate(sample_rate)
+                    wav_file.writeframes(audio_int16.tobytes())
+
+                logger.debug(f"Wrote audio to {filename}")
+            except Exception as e:
+                logger.error(f"Failed to write audio to {filename}: {e}")
+
+        # Start background thread to write the WAV file
+        thread = threading.Thread(target=write_wav, daemon=True)
+        thread.start()
+
+    def _resample_to_model_rate(self, audio_array: np.ndarray) -> np.ndarray:
+        """Resample audio to the model's expected sample rate (16 kHz).
+
+        Args:
+            audio_array: Audio data as a float32 numpy array.
+
+        Returns:
+            Resampled audio array at 16 kHz.
+        """
+        actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
+        if actual_rate == _MODEL_SAMPLE_RATE:
+            return audio_array
+
+        return soxr.resample(audio_array, actual_rate, _MODEL_SAMPLE_RATE, quality="VHQ")
+
     def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
         """Predict end-of-turn using local ONNX model."""
 
-        def truncate_audio_to_last_n_seconds(audio_array, n_seconds=8, sample_rate=16000):
+        def truncate_audio_to_last_n_seconds(
+            audio_array, n_seconds=8, sample_rate=_MODEL_SAMPLE_RATE
+        ):
             """Truncate audio to last n seconds or pad with zeros to meet n seconds."""
             max_samples = n_seconds * sample_rate
             if len(audio_array) > max_samples:
@@ -95,16 +154,22 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
                 return np.pad(audio_array, (padding, 0), mode="constant", constant_values=0)
             return audio_array
 
+        audio_for_logging = audio_array
+        actual_rate = self._sample_rate or _MODEL_SAMPLE_RATE
+
+        # Resample to 16 kHz if the pipeline uses a different sample rate
+        audio_array = self._resample_to_model_rate(audio_array)
+
         # Truncate to 8 seconds (keeping the end) or pad to 8 seconds
         audio_array = truncate_audio_to_last_n_seconds(audio_array, n_seconds=8)
 
         # Process audio using Whisper's feature extractor
         inputs = self._feature_extractor(
             audio_array,
-            sampling_rate=16000,
+            sampling_rate=_MODEL_SAMPLE_RATE,
             return_tensors="np",
             padding="max_length",
-            max_length=8 * 16000,
+            max_length=8 * _MODEL_SAMPLE_RATE,
             truncation=True,
             do_normalize=True,
         )
@@ -122,6 +187,10 @@ class LocalSmartTurnAnalyzerV3(BaseSmartTurn):
         # Make prediction (1 for Complete, 0 for Incomplete)
         prediction = 1 if probability > 0.5 else 0
 
+        if self._log_data:
+            suffix = "_complete" if prediction == 1 else "_incomplete"
+            self._write_audio_to_wav(audio_for_logging, sample_rate=actual_rate, suffix=suffix)
+
         return {
             "prediction": prediction,
             "probability": probability,

src/pipecat/audio/vad/aic_vad.py

@@ -1,44 +1,44 @@
 """AIC-integrated VAD analyzer that lazily binds to the AIC SDK backend.
 
-This analyzer queries the backend's is_speech_detected() and maps it to a float
-confidence (1.0/0.0). It uses 10 ms windows based on the sample rate and applies
-optional AIC VAD parameters (lookback_buffer_size, sensitivity) when available.
+This module provides VAD analyzer implementations that query the AIC SDK's
+is_speech_detected() and map it to a float confidence (1.0/0.0).
+
+Classes:
+    AICVADAnalyzer: For aic-sdk (uses 'aic_sdk' module)
 """
 
 from typing import Any, Callable, Optional
 
+from aic_sdk import VadParameter
 from loguru import logger
 
 from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams
 
-try:
-    from aic import AICVadParameter
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error("In order to use the AIC filter, you need to `pip install pipecat-ai[aic]`.")
-    raise Exception(f"Missing module: {e}")
-
 
 class AICVADAnalyzer(VADAnalyzer):
-    """VAD analyzer that lazily instantiates the AIC VoiceActivityDetector via a factory.
+    """VAD analyzer that lazily binds to the AIC VadContext via a factory.
 
-    The analyzer can be constructed before the AIC Model exists. Once the filter has
-    started and the Model is available, the provided factory will succeed and the
-    backend VAD will be created. We then switch to single-sample updates where
-    num_frames_required() returns 1 and confidence is derived from the backend's
-    boolean is_speech_detected() state.
+    The analyzer can be constructed before the AIC Processor exists. Once the filter has
+    started and the Processor is available, the provided factory will succeed and the
+    VadContext will be obtained. The context's is_speech_detected() boolean state is
+    then mapped to 1.0 (speech) or 0.0 (no speech) to satisfy the VADAnalyzer interface.
 
     AIC VAD runtime parameters:
-      - lookback_buffer_size:
-          Controls the lookback buffer size used by the VAD, i.e. the number of
-          window-length audio buffers used as a lookback buffer. Larger values improve
-          stability but increase latency.
-          Range: 1.0 .. 20.0
-          Default (SDK): 6.0
+      - speech_hold_duration:
+          Controls for how long the VAD continues to detect speech after the audio signal
+          no longer contains speech (in seconds).
+          Range: 0.0 to 100x model window length
+          Default (SDK): 0.05s (50ms)
+      - minimum_speech_duration:
+          Controls for how long speech needs to be present in the audio signal before the
+          VAD considers it speech (in seconds).
+          Range: 0.0 to 1.0
+          Default (SDK): 0.0s
       - sensitivity:
-          Controls the energy threshold sensitivity. Higher values make the detector
-          less sensitive (require more energy to count as speech).
-          Range: 1.0 .. 15.0
+          Controls the sensitivity (energy threshold) of the VAD. This value is used by
+          the VAD as the threshold a speech audio signal's energy has to exceed in order
+          to be considered speech.
+          Range: 1.0 to 15.0
           Formula: Energy threshold = 10 ** (-sensitivity)
           Default (SDK): 6.0
     """
@@ -46,69 +46,80 @@ class AICVADAnalyzer(VADAnalyzer):
     def __init__(
         self,
         *,
-        vad_factory: Optional[Callable[[], Any]] = None,
-        lookback_buffer_size: Optional[float] = None,
+        vad_context_factory: Optional[Callable[[], Any]] = None,
+        speech_hold_duration: Optional[float] = None,
+        minimum_speech_duration: Optional[float] = None,
         sensitivity: Optional[float] = None,
     ):
         """Create an AIC VAD analyzer.
 
         Args:
-            vad_factory:
-                Zero-arg callable that returns an initialized AIC VoiceActivityDetector.
-                This may raise until the filter's Model has been created; the analyzer
+            vad_context_factory:
+                Zero-arg callable that returns the AIC VadContext.
+                This may raise until the filter's Processor has been created; the analyzer
                 will retry on set_sample_rate/first use.
-            lookback_buffer_size:
-                Optional override for AIC VAD lookback buffer size.
-                Range: 1.0 .. 20.0. Larger values increase stability at the cost of latency.
-                If None, the SDK default (6.0) is used.
+            speech_hold_duration:
+                Optional override for AIC VAD speech hold duration (in seconds).
+                Range: 0.0 to 100x model window length.
+                If None, the SDK default (0.05s) is used.
+            minimum_speech_duration:
+                Optional override for minimum speech duration before VAD reports
+                speech detected (in seconds).
+                Range: 0.0 to 1.0.
+                If None, the SDK default (0.0s) is used.
             sensitivity:
                 Optional override for AIC VAD sensitivity (energy threshold).
-                Range: 1.0 .. 15.0. Energy threshold = 10 ** (-sensitivity).
+                Range: 1.0 to 15.0. Energy threshold = 10 ** (-sensitivity).
                 If None, the SDK default (6.0) is used.
         """
         # Use fixed VAD parameters for AIC: no user override
         fixed_params = VADParams(confidence=0.5, start_secs=0.0, stop_secs=0.0, min_volume=0.0)
         super().__init__(sample_rate=None, params=fixed_params)
-        self._vad_factory = vad_factory
-        self._backend_vad: Optional[Any] = None
-        self._pending_lookback: Optional[float] = lookback_buffer_size
+
+        self._vad_context_factory = vad_context_factory
+        self._vad_ctx: Optional[Any] = None
+        self._pending_speech_hold_duration: Optional[float] = speech_hold_duration
+        self._pending_minimum_speech_duration: Optional[float] = minimum_speech_duration
         self._pending_sensitivity: Optional[float] = sensitivity
 
-    def bind_vad_factory(self, vad_factory: Callable[[], Any]):
+    def bind_vad_context_factory(self, vad_context_factory: Callable[[], Any]):
         """Attach or replace the factory post-construction."""
-        self._vad_factory = vad_factory
-        self._ensure_backend_initialized()
+        self._vad_context_factory = vad_context_factory
+        self._ensure_vad_context_initialized()
 
-    def _apply_backend_params(self):
+    def _apply_vad_params(self):
         """Apply optional AIC VAD parameters if available."""
-        if self._backend_vad is None or AICVadParameter is None:
+        if self._vad_ctx is None or VadParameter is None:
             return
+
         try:
-            if self._pending_lookback is not None:
-                self._backend_vad.set_parameter(
-                    AICVadParameter.LOOKBACK_BUFFER_SIZE, float(self._pending_lookback)
+            if self._pending_speech_hold_duration is not None:
+                self._vad_ctx.set_parameter(
+                    VadParameter.SpeechHoldDuration, self._pending_speech_hold_duration
+                )
+            if self._pending_minimum_speech_duration is not None:
+                self._vad_ctx.set_parameter(
+                    VadParameter.MinimumSpeechDuration, self._pending_minimum_speech_duration
                 )
             if self._pending_sensitivity is not None:
-                self._backend_vad.set_parameter(
-                    AICVadParameter.SENSITIVITY, float(self._pending_sensitivity)
-                )
+                self._vad_ctx.set_parameter(VadParameter.Sensitivity, self._pending_sensitivity)
         except Exception as e:  # noqa: BLE001
             logger.debug(f"AIC VAD parameter application deferred/failed: {e}")
 
-    def _ensure_backend_initialized(self):
-        if self._backend_vad is not None:
+    def _ensure_vad_context_initialized(self):
+        if self._vad_ctx is not None:
             return
-        if not self._vad_factory:
+        if not self._vad_context_factory:
             return
         try:
-            self._backend_vad = self._vad_factory()
-            self._apply_backend_params()
-            # With backend ready, recompute internal frame sizing
+            self._vad_ctx = self._vad_context_factory()
+            self._apply_vad_params()
+            # With VAD context ready, recompute internal frame sizing
             super().set_params(self._params)
-            logger.debug("AIC VAD backend initialized in analyzer.")
+            logger.debug("AIC VAD context initialized in analyzer.")
         except Exception as e:  # noqa: BLE001
             # Filter may not be started yet; try again later
-            logger.debug(f"Deferring AIC VAD backend initialization: {e}")
+            logger.debug(f"Deferring AIC VAD context initialization: {e}")
 
     def set_sample_rate(self, sample_rate: int):
         """Set the sample rate for audio processing.
@@ -116,10 +127,10 @@ class AICVADAnalyzer(VADAnalyzer):
         Args:
             sample_rate: Audio sample rate in Hz.
         """
-        # Set rate and attempt backend initialization once we know SR
+        # Set rate and attempt VAD context initialization once we know SR
         self._sample_rate = self._init_sample_rate or sample_rate
-        self._ensure_backend_initialized()
-        # Ensure params are initialized even if backend not ready yet
+        self._ensure_vad_context_initialized()
+        # Ensure params are initialized even if VAD context not ready yet
         try:
             super().set_params(self._params)
         except Exception:
@@ -135,23 +146,29 @@ class AICVADAnalyzer(VADAnalyzer):
         return int(self.sample_rate * 0.01) if self.sample_rate > 0 else 160
 
     def voice_confidence(self, buffer: bytes) -> float:
-        """Calculate voice activity confidence for the given audio buffer.
+        """Return voice activity detection result for the given audio buffer.
+
+        Note:
+            The AIC SDK provides binary speech detection (not a probability score).
+            This method returns 1.0 when speech is detected and 0.0 otherwise,
+            rather than a true confidence value.
 
         Args:
-            buffer: Audio buffer to analyze.
+            buffer: Audio buffer (unused - AIC VAD state is updated internally
+                by the enhancement pipeline).
 
         Returns:
-            Voice confidence score is 0.0 or 1.0.
+            1.0 if speech is detected, 0.0 otherwise.
         """
-        # Ensure backend exists (filter might have started since last call)
-        self._ensure_backend_initialized()
-        if self._backend_vad is None:
+        # Ensure VAD context exists (filter might have started since last call)
+        self._ensure_vad_context_initialized()
+        if self._vad_ctx is None:
             return 0.0
 
-        # We do not need to analyze 'buffer' here since the model's VAD is updated
+        # We do not need to analyze 'buffer' here since the processor's VAD is updated
         # as part of the enhancement pipeline. Simply query the boolean and map it.
         try:
-            is_speech = self._backend_vad.is_speech_detected()
+            is_speech = self._vad_ctx.is_speech_detected()
             return 1.0 if is_speech else 0.0
         except Exception as e:  # noqa: BLE001
             logger.error(f"AIC VAD inference error: {e}")

src/pipecat/audio/vad/silero.py

@@ -27,7 +27,7 @@ try:
 
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
-    logger.error("In order to use Silero VAD, you need to `pip install pipecat-ai[silero]`.")
+    logger.error("In order to use Silero VAD, you need to `pip install pipecat-ai`.")
     raise Exception(f"Missing module(s): {e}")
 
 

src/pipecat/audio/vad/vad_analyzer.py

@@ -24,7 +24,7 @@ from pipecat.audio.utils import calculate_audio_volume, exp_smoothing
 
 VAD_CONFIDENCE = 0.7
 VAD_START_SECS = 0.2
-VAD_STOP_SECS = 0.8
+VAD_STOP_SECS = 0.2
 VAD_MIN_VOLUME = 0.6
 
 
@@ -127,7 +127,7 @@ class VADAnalyzer(ABC):
         pass
 
     @abstractmethod
-    def voice_confidence(self, buffer) -> float:
+    def voice_confidence(self, buffer: bytes) -> float:
         """Calculate voice activity confidence for the given audio buffer.
 
         Args:

src/pipecat/audio/vad/vad_controller.py

@@ -0,0 +1,171 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Voice Activity Detection controller for managing speech state transitions.
+
+This module provides a controller that wraps a VADAnalyzer to track speech state
+and emit events when speech starts, stops, or is actively detected.
+"""
+
+import time
+from typing import Type
+
+from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
+from pipecat.frames.frames import (
+    Frame,
+    InputAudioRawFrame,
+    SpeechControlParamsFrame,
+    StartFrame,
+    VADParamsUpdateFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.utils.base_object import BaseObject
+
+
+class VADController(BaseObject):
+    """Manages voice activity detection state and emits speech events.
+
+    Wraps a `VADAnalyzer` to process audio and trigger events based on speech
+    state transitions. Tracks whether the user is speaking, quiet, or
+    transitioning between states.
+
+    Event handlers available:
+
+    - on_speech_started: Called when speech begins.
+    - on_speech_stopped: Called when speech ends.
+    - on_speech_activity: Called periodically while speech is detected.
+    - on_push_frame: Called when the controller wants to push a frame.
+    - on_broadcast_frame: Called when the controller wants to broadcast a frame.
+
+    Example::
+
+        @vad_controller.event_handler("on_speech_started")
+        async def on_speech_started(controller):
+            ...
+
+        @vad_controller.event_handler("on_speech_stopped")
+        async def on_speech_stopped(controller):
+            ...
+
+        @vad_controller.event_handler("on_speech_activity")
+        async def on_speech_activity(controller):
+            ...
+
+        @vad_controller.event_handler("on_push_frame")
+        async def on_push_frame(controller, frame: Frame, direction: FrameDirection):
+            ...
+
+        @vad_controller.event_handler("on_broadcast_frame")
+        async def on_broadcast_frame(controller, frame_cls: Type[Frame], **kwargs):
+            ...
+    """
+
+    def __init__(self, vad_analyzer: VADAnalyzer, *, speech_activity_period: float = 0.2):
+        """Initialize the VAD controller.
+
+        Args:
+            vad_analyzer: The `VADAnalyzer` instance for processing audio.
+            speech_activity_period: Minimum interval in seconds between
+                `on_speech_activity` events. Defaults to 0.2.
+        """
+        super().__init__()
+        self._vad_analyzer = vad_analyzer
+        self._vad_state: VADState = VADState.QUIET
+
+        # Last time a on_speech_activity was triggered.
+        self._speech_activity_time = 0
+        # How often a on_speech_activity event should be triggered (value should
+        # be greater than the audio chunks to have any effect).
+        self._speech_activity_period = speech_activity_period
+
+        self._register_event_handler("on_speech_started", sync=True)
+        self._register_event_handler("on_speech_stopped", sync=True)
+        self._register_event_handler("on_speech_activity", sync=True)
+        self._register_event_handler("on_push_frame", sync=True)
+        self._register_event_handler("on_broadcast_frame", sync=True)
+
+    async def process_frame(self, frame: Frame):
+        """Process a frame and handle VAD-related events.
+
+        Handles `StartFrame` to initialize the sample rate and `InputAudioRawFrame`
+        to analyze audio for voice activity.
+
+        Args:
+            frame: The frame to process.
+        """
+        if isinstance(frame, StartFrame):
+            await self._start(frame)
+        elif isinstance(frame, InputAudioRawFrame):
+            await self._handle_audio(frame)
+        elif isinstance(frame, VADParamsUpdateFrame):
+            self._vad_analyzer.set_params(frame.params)
+            await self.broadcast_frame(SpeechControlParamsFrame, vad_params=frame.params)
+
+    async def _start(self, frame: StartFrame):
+        self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate)
+        # Broadcast initial VAD params so other services (e.g. STT) can use them
+        await self.broadcast_frame(SpeechControlParamsFrame, vad_params=self._vad_analyzer.params)
+
+    async def _handle_audio(self, frame: InputAudioRawFrame):
+        """Process an audio chunk and emit speech events as needed.
+
+        Analyzes the audio for voice activity and triggers `on_speech_started`,
+        `on_speech_stopped`, or `on_speech_activity` events based on state changes.
+
+        Args:
+            frame: Audio frame to process.
+        """
+        self._vad_state = await self._handle_vad(frame.audio, self._vad_state)
+
+        if self._vad_state == VADState.SPEAKING:
+            await self._call_event_handler("on_speech_activity")
+
+    async def _handle_vad(self, audio: bytes, vad_state: VADState) -> VADState:
+        """Handle Voice Activity Detection results and trigger appropriate events."""
+        new_vad_state = await self._vad_analyzer.analyze_audio(audio)
+        if (
+            new_vad_state != vad_state
+            and new_vad_state != VADState.STARTING
+            and new_vad_state != VADState.STOPPING
+        ):
+            if new_vad_state == VADState.SPEAKING:
+                await self._call_event_handler("on_speech_started")
+            elif new_vad_state == VADState.QUIET:
+                await self._call_event_handler("on_speech_stopped")
+
+            vad_state = new_vad_state
+        return vad_state
+
+    async def _maybe_speech_activity(self):
+        """Handle user speaking frame."""
+        diff_time = time.time() - self._speech_activity_time
+        if diff_time >= self._speech_activity_period:
+            self._speech_activity_time = time.time()
+            await self._call_event_handler("on_speech_activity")
+
+    async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+        """Request a frame to be pushed through the pipeline.
+
+        This emits an on_push_frame event that must be handled by a processor
+        to actually push the frame into the pipeline.
+
+        Args:
+            frame: The frame to push.
+            direction: The direction to push the frame.
+        """
+        await self._call_event_handler("on_push_frame", frame, direction)
+
+    async def broadcast_frame(self, frame_cls: Type[Frame], **kwargs):
+        """Request a frame to be broadcast upstream and downstream.
+
+        This emits an on_broadcast_frame event that must be handled by a processor
+        to actually broadcast the frame in the pipeline.
+
+        Args:
+            frame_cls: The class of the frame to broadcast.
+            **kwargs: Arguments to pass to the frame constructor.
+        """
+        await self._call_event_handler("on_broadcast_frame", frame_cls, **kwargs)

src/pipecat/extensions/ivr/ivr_navigator.py

@@ -18,6 +18,7 @@ from loguru import logger
 from pipecat.audio.dtmf.types import KeypadEntry
 from pipecat.audio.vad.vad_analyzer import VADParams
 from pipecat.frames.frames import (
+    AggregatedTextFrame,
     EndFrame,
     Frame,
     LLMContextFrame,
@@ -153,13 +154,19 @@ class IVRProcessor(FrameProcessor):
             # Process text through the pattern aggregator
             async for result in self._aggregator.aggregate(frame.text):
                 # Push aggregated text that doesn't contain XML patterns
-                await self.push_frame(LLMTextFrame(result.text), direction)
+                await self.push_frame(
+                    AggregatedTextFrame(text=result.text, aggregated_by=result.type),
+                    direction,
+                )
 
         elif isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
             # Flush any remaining text from the aggregator
             remaining = await self._aggregator.flush()
             if remaining:
-                await self.push_frame(LLMTextFrame(remaining.text), direction)
+                await self.push_frame(
+                    AggregatedTextFrame(text=remaining.text, aggregated_by=remaining.type),
+                    direction,
+                )
             # Push the end frame
             await self.push_frame(frame, direction)
 

src/pipecat/extensions/voicemail/voicemail_detector.py

@@ -368,7 +368,7 @@ class ClassificationProcessor(FrameProcessor):
             await self._voicemail_notifier.notify()  # Clear buffered TTS frames
 
             # Interrupt the current pipeline to stop any ongoing processing
-            await self.push_interruption_task_frame_and_wait()
+            await self.broadcast_interruption()
 
             # Set the voicemail event to trigger the voicemail handler
             self._voicemail_event.clear()

src/pipecat/frames/frames.py

@@ -11,8 +11,8 @@ including data frames, system frames, and control frames for audio, video, text,
 and LLM processing.
 """
 
+import time
 from dataclasses import dataclass, field
-from enum import Enum
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -34,12 +34,16 @@ from pipecat.audio.turn.base_turn_analyzer import BaseTurnParams
 from pipecat.audio.vad.vad_analyzer import VADParams
 from pipecat.metrics.metrics import MetricsData
 from pipecat.transcriptions.language import Language
+from pipecat.utils.text.base_text_aggregator import AggregationType
 from pipecat.utils.time import nanoseconds_to_str
 from pipecat.utils.utils import obj_count, obj_id
 
 if TYPE_CHECKING:
     from pipecat.processors.aggregators.llm_context import LLMContext, NotGiven
     from pipecat.processors.frame_processor import FrameProcessor
+    from pipecat.services.settings import ServiceSettings
+    from pipecat.utils.context.llm_context_summarization import LLMContextSummaryConfig
+    from pipecat.utils.tracing.tracing_context import TracingContext
 
 
 class DeprecatedKeypadEntry:
@@ -120,6 +124,9 @@ class Frame:
         id: Unique identifier for the frame instance.
         name: Human-readable name combining class name and instance count.
         pts: Presentation timestamp in nanoseconds.
+        broadcast_sibling_id: ID of the paired frame when this frame was
+            broadcast in both directions. Set automatically by
+            ``broadcast_frame()`` and ``broadcast_frame_instance()``.
         metadata: Dictionary for arbitrary frame metadata.
         transport_source: Name of the transport source that created this frame.
         transport_destination: Name of the transport destination for this frame.
@@ -128,6 +135,7 @@ class Frame:
     id: int = field(init=False)
     name: str = field(init=False)
     pts: Optional[int] = field(init=False)
+    broadcast_sibling_id: Optional[int] = field(init=False)
     metadata: Dict[str, Any] = field(init=False)
     transport_source: Optional[str] = field(init=False)
     transport_destination: Optional[str] = field(init=False)
@@ -136,6 +144,7 @@ class Frame:
         self.id: int = obj_id()
         self.name: str = f"{self.__class__.__name__}#{obj_count(self)}"
         self.pts: Optional[int] = None
+        self.broadcast_sibling_id: Optional[int] = None
         self.metadata: Dict[str, Any] = {}
         self.transport_source: Optional[str] = None
         self.transport_destination: Optional[str] = None
@@ -265,8 +274,16 @@ class OutputImageRawFrame(DataFrame, ImageRawFrame):
     An image that will be shown by the transport. If the transport supports
     multiple video destinations (e.g. multiple video tracks) the destination
     name can be specified in transport_destination.
+
+    Parameters:
+        sync_with_audio: If True, the image is queued with audio frames so
+            it is only displayed after all preceding audio has been sent.
+            Defaults to False (image is displayed immediately when the output
+            transport receives it).
     """
 
+    sync_with_audio: bool = field(default=False, init=False)
+
     def __str__(self):
         pts = format_pts(self.pts)
         return f"{self.name}(pts: {pts}, destination: {self.transport_destination}, size: {self.size}, format: {self.format})"
@@ -277,9 +294,12 @@ class TTSAudioRawFrame(OutputAudioRawFrame):
     """Audio data frame generated by Text-to-Speech services.
 
     A chunk of output audio generated by a TTS service, ready for playback.
+
+    Parameters:
+        context_id: Unique identifier for the TTS context that generated this audio.
     """
 
-    pass
+    context_id: Optional[str] = None
 
 
 @dataclass
@@ -341,6 +361,11 @@ class TextFrame(DataFrame):
 
     Parameters:
         text: The text content.
+        skip_tts: Whether this text should be skipped by the TTS service.
+        includes_inter_frame_spaces: Whether any necessary inter-frame (leading/trailing) spaces are already
+            included in the text.
+        append_to_context: Whether this text should be appended to the LLM context.
+            Defaults to True.
     """
 
     text: str
@@ -376,16 +401,6 @@ class LLMTextFrame(TextFrame):
         self.includes_inter_frame_spaces = True
 
 
-class AggregationType(str, Enum):
-    """Built-in aggregation strings."""
-
-    SENTENCE = "sentence"
-    WORD = "word"
-
-    def __str__(self):
-        return self.value
-
-
 @dataclass
 class AggregatedTextFrame(TextFrame):
     """Text frame representing an aggregation of TextFrames.
@@ -395,9 +410,11 @@ class AggregatedTextFrame(TextFrame):
 
     Parameters:
         aggregated_by: Method used to aggregate the text frames.
+        context_id: Unique identifier for the TTS context that generated this text.
     """
 
     aggregated_by: AggregationType | str
+    context_id: Optional[str] = None
 
 
 @dataclass
@@ -409,9 +426,13 @@ class VisionTextFrame(LLMTextFrame):
 
 @dataclass
 class TTSTextFrame(AggregatedTextFrame):
-    """Text frame generated by Text-to-Speech services."""
+    """Text frame generated by Text-to-Speech services.
 
-    pass
+    Parameters:
+        context_id: Unique identifier for the TTS context that generated this text.
+    """
+
+    context_id: Optional[str] = None
 
 
 @dataclass
@@ -426,12 +447,15 @@ class TranscriptionFrame(TextFrame):
         timestamp: When the transcription occurred.
         language: Detected or specified language of the speech.
         result: Raw result from the STT service.
+        finalized: Whether this is the final transcription for an utterance.
+            Set by STT services that support commit/finalize signals.
     """
 
     user_id: str
     timestamp: str
     language: Optional[Language] = None
     result: Optional[Any] = None
+    finalized: bool = False
 
     def __str__(self):
         return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})"
@@ -918,9 +942,11 @@ class TTSSpeakFrame(DataFrame):
 
     Parameters:
         text: The text to be spoken.
+        append_to_context: Whether to append the text to the context.
     """
 
     text: str
+    append_to_context: Optional[bool] = None
 
 
 @dataclass
@@ -983,7 +1009,8 @@ class OutputDTMFFrame(DTMFFrame, DataFrame):
     specify where the DTMF keypress should be sent.
     """
 
-    pass
+    def __str__(self):
+        return f"{self.name}(tone: {self.button})"
 
 
 #
@@ -1015,6 +1042,7 @@ class StartFrame(SystemFrame):
                 Use  `LLMUserAggregator`'s new `user_turn_strategies` parameter instead.
 
         report_only_initial_ttfb: Whether to report only initial time-to-first-byte.
+        tracing_context: Pipeline-scoped tracing context for span hierarchy.
     """
 
     audio_in_sample_rate: int = 16000
@@ -1025,6 +1053,7 @@ class StartFrame(SystemFrame):
     enable_usage_metrics: bool = False
     interruption_strategies: List[BaseInterruptionStrategy] = field(default_factory=list)
     report_only_initial_ttfb: bool = False
+    tracing_context: Optional["TracingContext"] = None
 
 
 @dataclass
@@ -1115,12 +1144,11 @@ class FrameProcessorResumeUrgentFrame(SystemFrame):
 
 @dataclass
 class InterruptionFrame(SystemFrame):
-    """Frame indicating user started speaking (interruption detected).
+    """Frame pushed to interrupt the pipeline.
 
-    Emitted by the BaseInputTransport to indicate that a user has started
-    speaking (i.e. is interrupting). This is similar to
-    UserStartedSpeakingFrame except that it should be pushed concurrently
-    with other frames (so the order is not guaranteed).
+    This frame is used to interrupt the pipeline. For example, when a user
+    starts speaking to cancel any in-progress bot output. It can also be pushed
+    by any processor.
     """
 
     pass
@@ -1190,6 +1218,28 @@ class UserStoppedSpeakingFrame(SystemFrame):
     emulated: bool = False
 
 
+@dataclass
+class UserMuteStartedFrame(SystemFrame):
+    """Frame indicating that the user has been muted.
+
+    Emitted when a mute strategy activates, suppressing user frames (audio,
+    transcription, interruption) from propagating through the pipeline.
+    """
+
+    pass
+
+
+@dataclass
+class UserMuteStoppedFrame(SystemFrame):
+    """Frame indicating that the user has been unmuted.
+
+    Emitted when a mute strategy deactivates, allowing user frames to
+    propagate through the pipeline again.
+    """
+
+    pass
+
+
 @dataclass
 class UserSpeakingFrame(SystemFrame):
     """Frame indicating the user is speaking.
@@ -1252,16 +1302,32 @@ class EmulateUserStoppedSpeakingFrame(SystemFrame):
 
 @dataclass
 class VADUserStartedSpeakingFrame(SystemFrame):
-    """Frame emitted when VAD definitively detects user started speaking."""
+    """Frame emitted when VAD definitively detects user started speaking.
 
-    pass
+    Parameters:
+        start_secs: The VAD start_secs duration that was used to confirm the user
+            started speaking. This represents the speech duration that had to
+            elapse before the VAD determined speech began.
+        timestamp: Wall-clock time when the VAD made its determination.
+    """
+
+    start_secs: float = 0.0
+    timestamp: float = field(default_factory=time.time)
 
 
 @dataclass
 class VADUserStoppedSpeakingFrame(SystemFrame):
-    """Frame emitted when VAD definitively detects user stopped speaking."""
+    """Frame emitted when VAD definitively detects user stopped speaking.
 
-    pass
+    Parameters:
+        stop_secs: The VAD stop_secs duration that was used to confirm the user
+            stopped speaking. This represents the silence duration that had to
+            elapse before the VAD determined speech ended.
+        timestamp: Wall-clock time when the VAD made its determination.
+    """
+
+    stop_secs: float = 0.0
+    timestamp: float = field(default_factory=time.time)
 
 
 @dataclass
@@ -1461,29 +1527,31 @@ class UserImageRequestFrame(SystemFrame):
         text: An optional text associated to the image request.
         append_to_context: Whether the requested image should be appended to the LLM context.
         video_source: Specific video source to capture from.
+        function_name: Name of function that generated this request (if any).
+        tool_call_id: Tool call ID if generated by function call (if any).
+        result_callback: Optional callback to invoke when the image is retrieved.
         context: [DEPRECATED] Optional context for the image request.
-        function_name: [DEPRECATED] Name of function that generated this request (if any).
-        tool_call_id: [DEPRECATED] Tool call ID if generated by function call.
     """
 
     user_id: str
     text: Optional[str] = None
     append_to_context: Optional[bool] = None
     video_source: Optional[str] = None
-    context: Optional[Any] = None
     function_name: Optional[str] = None
     tool_call_id: Optional[str] = None
+    result_callback: Optional[Any] = None
+    context: Optional[Any] = None
 
     def __post_init__(self):
         super().__post_init__()
 
-        if self.context or self.function_name or self.tool_call_id:
+        if self.context:
             import warnings
 
             with warnings.catch_warnings():
                 warnings.simplefilter("always")
                 warnings.warn(
-                    "`UserImageRequestFrame` fields `context`, `function_name` and `tool_call_id` are deprecated.",
+                    "`UserImageRequestFrame` field `context` is deprecated.",
                     DeprecationWarning,
                     stacklevel=2,
                 )
@@ -1565,7 +1633,7 @@ class UserImageRawFrame(InputImageRawFrame):
         user_id: Identifier of the user who provided this image.
         text: An optional text associated to this image.
         append_to_context: Whether the requested image should be appended to the LLM context.
-        request: [DEPRECATED] The original image request frame if this is a response.
+        request: The original image request frame if this is a response.
     """
 
     user_id: str = ""
@@ -1573,20 +1641,6 @@ class UserImageRawFrame(InputImageRawFrame):
     append_to_context: Optional[bool] = None
     request: Optional[UserImageRequestFrame] = None
 
-    def __post_init__(self):
-        super().__post_init__()
-
-        if self.request:
-            import warnings
-
-            with warnings.catch_warnings():
-                warnings.simplefilter("always")
-                warnings.warn(
-                    "`UserImageRawFrame` field `request` is deprecated.",
-                    DeprecationWarning,
-                    stacklevel=2,
-                )
-
     def __str__(self):
         pts = format_pts(self.pts)
         return f"{self.name}(pts: {pts}, user: {self.user_id}, source: {self.transport_source}, size: {self.size}, format: {self.format}, text: {self.text}, append_to_context: {self.append_to_context})"
@@ -1613,7 +1667,8 @@ class AssistantImageRawFrame(OutputImageRawFrame):
 class InputDTMFFrame(DTMFFrame, SystemFrame):
     """DTMF keypress input frame from transport."""
 
-    pass
+    def __str__(self):
+        return f"{self.name}(tone: {self.button.value})"
 
 
 @dataclass
@@ -1645,13 +1700,59 @@ class SpeechControlParamsFrame(SystemFrame):
     turn_params: Optional[BaseTurnParams] = None
 
 
+@dataclass
+class ServiceMetadataFrame(SystemFrame):
+    """Base metadata frame for services.
+
+    Broadcast by services at pipeline start to share service-specific
+    configuration and performance characteristics with downstream processors.
+
+    Parameters:
+        service_name: The name of the service broadcasting this metadata.
+    """
+
+    service_name: str
+
+
+@dataclass
+class STTMetadataFrame(ServiceMetadataFrame):
+    """Metadata from STT service.
+
+    Broadcast by STT services to inform downstream processors (like turn
+    strategies) about STT latency characteristics.
+
+    Parameters:
+        ttfs_p99_latency: Time to final segment P99 latency in seconds.
+            This is the expected time from when speech ends to when the
+            final transcript is received, at the 99th percentile.
+    """
+
+    ttfs_p99_latency: float
+
+
+@dataclass
+class ServiceSwitcherRequestMetadataFrame(ControlFrame):
+    """Request a service to re-emit its metadata frames.
+
+    Used by ServiceSwitcher when switching active services to ensure
+    downstream processors receive updated metadata from the newly active service.
+    Services that receive this frame should re-push their metadata frame
+    (e.g., STTMetadataFrame for STT services).
+
+    Parameters:
+        service: The target service that should re-emit its metadata.
+    """
+
+    service: "FrameProcessor"
+
+
 #
 # Task frames
 #
 
 
 @dataclass
-class TaskFrame(SystemFrame):
+class TaskFrame(ControlFrame):
     """Base frame for task frames.
 
     This is a base class for frames that are meant to be sent and handled
@@ -1665,7 +1766,21 @@ class TaskFrame(SystemFrame):
 
 
 @dataclass
-class EndTaskFrame(TaskFrame):
+class TaskSystemFrame(SystemFrame):
+    """Base frame for task system frames.
+
+    This is a base class for frames that are meant to be sent and handled
+    upstream by the pipeline task. This might result in a corresponding frame
+    sent downstream (e.g. `InterruptionTaskFrame` / `InterruptionFrame` or
+    `EndTaskFrame` / `EndFrame`).
+
+    """
+
+    pass
+
+
+@dataclass
+class EndTaskFrame(TaskFrame, UninterruptibleFrame):
     """Frame to request graceful pipeline task closure.
 
     This is used to notify the pipeline task that the pipeline should be
@@ -1683,7 +1798,20 @@ class EndTaskFrame(TaskFrame):
 
 
 @dataclass
-class CancelTaskFrame(TaskFrame):
+class StopTaskFrame(TaskFrame, UninterruptibleFrame):
+    """Frame to request pipeline task stop while keeping processors running.
+
+    This is used to notify the pipeline task that it should be stopped as
+    soon as possible (flushing all the queued frames) but that the pipeline
+    processors should be kept in a running state. This frame should be pushed
+    upstream.
+    """
+
+    pass
+
+
+@dataclass
+class CancelTaskFrame(TaskSystemFrame):
     """Frame to request immediate pipeline task cancellation.
 
     This is used to notify the pipeline task that the pipeline should be
@@ -1701,26 +1829,12 @@ class CancelTaskFrame(TaskFrame):
 
 
 @dataclass
-class StopTaskFrame(TaskFrame):
-    """Frame to request pipeline task stop while keeping processors running.
+class InterruptionTaskFrame(TaskSystemFrame):
+    """Frame indicating the pipeline should be interrupted.
 
-    This is used to notify the pipeline task that it should be stopped as
-    soon as possible (flushing all the queued frames) but that the pipeline
-    processors should be kept in a running state. This frame should be pushed
-    upstream.
-    """
-
-    pass
-
-
-@dataclass
-class InterruptionTaskFrame(TaskFrame):
-    """Frame indicating the bot should be interrupted.
-
-    Emitted when the bot should be interrupted. This will mainly cause the
-    same actions as if the user interrupted except that the
-    UserStartedSpeakingFrame and UserStoppedSpeakingFrame won't be generated.
-    This frame should be pushed upstream.
+    This frame should be pushed upstream to indicate the pipeline should be
+    interrupted. The pipeline task converts this into an `InterruptionFrame`
+    and sends it downstream.
     """
 
     pass
@@ -1760,7 +1874,7 @@ class BotInterruptionFrame(InterruptionTaskFrame):
 
 
 @dataclass
-class EndFrame(ControlFrame):
+class EndFrame(ControlFrame, UninterruptibleFrame):
     """Frame indicating pipeline has ended and should shut down.
 
     Indicates that a pipeline has ended and frame processors and pipelines
@@ -1769,6 +1883,10 @@ class EndFrame(ControlFrame):
     that this is a control frame, which means it will be received in the order it
     was sent.
 
+    This frame is marked as UninterruptibleFrame to ensure it is not lost when
+    an InterruptionFrame is processed. Terminal frames must survive interruption
+    to guarantee proper pipeline shutdown.
+
     Parameters:
         reason: Optional reason for pushing an end frame.
     """
@@ -1780,12 +1898,39 @@ class EndFrame(ControlFrame):
 
 
 @dataclass
-class StopFrame(ControlFrame):
+class StopFrame(ControlFrame, UninterruptibleFrame):
     """Frame indicating pipeline should stop but keep processors running.
 
     Indicates that a pipeline should be stopped but that the pipeline
     processors should be kept in a running state. This is normally queued from
     the pipeline task.
+
+    This frame is marked as UninterruptibleFrame to ensure it is not lost when
+    an InterruptionFrame is processed. Terminal frames must survive interruption
+    to guarantee proper pipeline control.
+    """
+
+    pass
+
+
+@dataclass
+class BotConnectedFrame(SystemFrame):
+    """Frame indicating the bot has connected to the transport service.
+
+    Pushed downstream by SFU transports (Daily, LiveKit, HeyGen, Tavus)
+    when the bot successfully joins the room. Non-SFU transports do not
+    emit this frame.
+    """
+
+    pass
+
+
+@dataclass
+class ClientConnectedFrame(SystemFrame):
+    """Frame indicating that a client has connected to the transport.
+
+    Pushed downstream by the input transport when a client (participant)
+    connects. Used by observers to measure transport readiness timing.
     """
 
     pass
@@ -1872,6 +2017,85 @@ class LLMFullResponseEndFrame(ControlFrame):
         self.skip_tts = None
 
 
+@dataclass
+class LLMAssistantPushAggregationFrame(ControlFrame):
+    """Frame that forces the LLM assistant aggregator to push its current aggregation to context.
+
+    When received by ``LLMAssistantAggregator``, any text that has been accumulated
+    in the aggregation buffer is immediately committed to the conversation context as
+    an assistant message, without waiting for an ``LLMFullResponseEndFrame``.
+    """
+
+
+@dataclass
+class LLMSummarizeContextFrame(ControlFrame):
+    """Frame requesting on-demand context summarization.
+
+    Push this frame into the pipeline to trigger a manual context summarization.
+
+    Parameters:
+        config: Optional per-request override for summary generation settings
+            (prompt, token budget, messages to keep). If ``None``, the
+            summarizer's default :class:`~pipecat.utils.context.llm_context_summarization.LLMContextSummaryConfig`
+            is used.
+    """
+
+    config: Optional["LLMContextSummaryConfig"] = None
+
+
+@dataclass
+class LLMContextSummaryRequestFrame(ControlFrame):
+    """Frame requesting context summarization from an LLM service.
+
+    Sent by aggregators to LLM services when conversation context needs to be
+    compressed. The LLM service generates a summary of older messages while
+    preserving recent conversation history.
+
+    Parameters:
+        request_id: Unique identifier to match this request with its response.
+            Used to handle async responses and avoid race conditions.
+        context: The full LLM context containing all messages to analyze and summarize.
+        min_messages_to_keep: Number of recent messages to preserve uncompressed.
+            These messages will not be included in the summary.
+        target_context_tokens: Maximum token size for the generated summary. This value
+            is passed directly to the LLM as the max_tokens parameter when generating
+            the summary text.
+        summarization_prompt: System prompt instructing the LLM how to generate
+            the summary.
+        summarization_timeout: Maximum time in seconds for the LLM to generate a
+            summary. When None, a default timeout of 120s is applied.
+    """
+
+    request_id: str
+    context: "LLMContext"
+    min_messages_to_keep: int
+    target_context_tokens: int
+    summarization_prompt: str
+    summarization_timeout: Optional[float] = None
+
+
+@dataclass
+class LLMContextSummaryResultFrame(ControlFrame, UninterruptibleFrame):
+    """Frame containing the result of context summarization.
+
+    Sent by LLM services back to aggregators after generating a summary.
+    Contains the formatted summary message and metadata about what was summarized.
+
+    Parameters:
+        request_id: Identifier matching the original request. Used to correlate
+            async responses.
+        summary: The formatted summary message ready to be inserted into context.
+        last_summarized_index: Index (0-based) of the last message that was
+            included in the summary. Messages after this index are preserved.
+        error: Error message if summarization failed, None on success.
+    """
+
+    request_id: str
+    summary: str
+    last_summarized_index: int
+    error: Optional[str] = None
+
+
 @dataclass
 class FunctionCallInProgressFrame(ControlFrame, UninterruptibleFrame):
     """Frame signaling that a function call is currently executing.
@@ -1920,29 +2144,49 @@ class TTSStartedFrame(ControlFrame):
     TTSStoppedFrame. These frames can be used for aggregating audio frames in a
     transport to optimize the size of frames sent to the session, without
     needing to control this in the TTS service.
+
+    Parameters:
+        context_id: Unique identifier for this TTS context.
     """
 
-    pass
+    context_id: Optional[str] = None
 
 
 @dataclass
 class TTSStoppedFrame(ControlFrame):
-    """Frame indicating the end of a TTS response."""
+    """Frame indicating the end of a TTS response.
 
-    pass
+    Parameters:
+        context_id: Unique identifier for this TTS context.
+    """
+
+    context_id: Optional[str] = None
 
 
 @dataclass
-class ServiceUpdateSettingsFrame(ControlFrame):
+class ServiceUpdateSettingsFrame(ControlFrame, UninterruptibleFrame):
     """Base frame for updating service settings.
 
-    A control frame containing a request to update service settings.
+    Supports both a ``settings`` dict (for backward compatibility) and a
+    ``delta`` object.  When both are provided, ``delta`` takes precedence.
 
     Parameters:
         settings: Dictionary of setting name to value mappings.
+
+            .. deprecated:: 0.0.104
+                Use ``delta`` with a typed settings object instead.
+
+        delta: :class:`~pipecat.services.settings.ServiceSettings` delta-mode
+            object describing the fields to change.
+
+        service: Optional target service instance. When provided, only that
+            service will apply the settings; other services will forward the
+            frame unchanged.
     """
 
-    settings: Mapping[str, Any]
+    settings: Mapping[str, Any] = field(default_factory=dict)
+    delta: Optional["ServiceSettings"] = None
+    service: Optional["FrameProcessor"] = None
 
 
 @dataclass
@@ -1966,6 +2210,20 @@ class STTUpdateSettingsFrame(ServiceUpdateSettingsFrame):
     pass
 
 
+@dataclass
+class UserIdleTimeoutUpdateFrame(SystemFrame):
+    """Frame for updating the user idle timeout at runtime.
+
+    Setting timeout to 0 disables idle detection. Setting a positive value
+    enables it.
+
+    Parameters:
+        timeout: The new idle timeout in seconds. 0 disables idle detection.
+    """
+
+    timeout: float
+
+
 @dataclass
 class VADParamsUpdateFrame(ControlFrame):
     """Frame for updating VAD parameters.

src/pipecat/metrics/metrics.py

@@ -87,19 +87,44 @@ class TTSUsageMetricsData(MetricsData):
     value: int
 
 
-class SmartTurnMetricsData(MetricsData):
-    """Metrics data for smart turn predictions.
+class TextAggregationMetricsData(MetricsData):
+    """Text aggregation time metrics data.
+
+    Measures the time from the first LLM token to the first complete sentence,
+    representing the latency cost of sentence aggregation in the TTS pipeline.
+
+    Parameters:
+        value: Aggregation time in seconds.
+    """
+
+    value: float
+
+
+class TurnMetricsData(MetricsData):
+    """Metrics data for turn detection predictions.
 
     Parameters:
         is_complete: Whether the turn is predicted to be complete.
         probability: Confidence probability of the turn completion prediction.
-        inference_time_ms: Time taken for inference in milliseconds.
-        server_total_time_ms: Total server processing time in milliseconds.
-        e2e_processing_time_ms: End-to-end processing time in milliseconds.
+        e2e_processing_time_ms: End-to-end processing time in milliseconds,
+            measured from VAD speech-to-silence transition to turn completion.
     """
 
     is_complete: bool
     probability: float
-    inference_time_ms: float
-    server_total_time_ms: float
     e2e_processing_time_ms: float
+
+
+class SmartTurnMetricsData(TurnMetricsData):
+    """Metrics data for smart turn predictions.
+
+    .. deprecated:: 0.0.104
+        Use :class:`TurnMetricsData` instead. This class will be removed in a future version.
+
+    Parameters:
+        inference_time_ms: Time taken for inference in milliseconds.
+        server_total_time_ms: Total server processing time in milliseconds.
+    """
+
+    inference_time_ms: float = 0.0
+    server_total_time_ms: float = 0.0

src/pipecat/observers/base_observer.py

@@ -100,3 +100,11 @@ class BaseObserver(BaseObject):
             data: The event data containing details about the frame transfer.
         """
         pass
+
+    async def on_pipeline_started(self):
+        """Called when the pipeline has fully started.
+
+        Fired after the ``StartFrame`` has been processed by all processors
+        in the pipeline, including nested ``ParallelPipeline`` branches.
+        """
+        pass

src/pipecat/observers/loggers/metrics_log_observer.py

@@ -24,6 +24,7 @@ from pipecat.metrics.metrics import (
     SmartTurnMetricsData,
     TTFBMetricsData,
     TTSUsageMetricsData,
+    TurnMetricsData,
 )
 from pipecat.observers.base_observer import BaseObserver, FramePushed
 
@@ -37,7 +38,7 @@ class MetricsLogObserver(BaseObserver):
     - ProcessingMetricsData (General processing time)
     - LLMUsageMetricsData (Token usage statistics)
     - TTSUsageMetricsData (Text-to-Speech character counts)
-    - SmartTurnMetricsData (Turn prediction metrics)
+    - TurnMetricsData (Turn prediction metrics)
 
     This allows developers to track performance metrics, token usage,
     and other statistics throughout the pipeline.
@@ -70,6 +71,17 @@ class MetricsLogObserver(BaseObserver):
             **kwargs: Additional arguments passed to parent class.
         """
         super().__init__(**kwargs)
+        # Normalize deprecated types in include_metrics
+        if include_metrics and SmartTurnMetricsData in include_metrics:
+            import warnings
+
+            warnings.warn(
+                "SmartTurnMetricsData is deprecated in include_metrics, "
+                "use TurnMetricsData instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            include_metrics = (include_metrics - {SmartTurnMetricsData}) | {TurnMetricsData}
         self._include_metrics = include_metrics
         self._frames_seen = set()
 
@@ -144,8 +156,8 @@ class MetricsLogObserver(BaseObserver):
             logger.debug(
                 f"📊 {processor_info} TTS USAGE{model_info}: {metrics_data.value} characters at {time_sec:.3f}s"
             )
-        elif isinstance(metrics_data, SmartTurnMetricsData):
-            self._log_smart_turn(metrics_data, processor_info, model_info, time_sec)
+        elif isinstance(metrics_data, TurnMetricsData):
+            self._log_turn(metrics_data, processor_info, model_info, time_sec)
         else:
             # Generic fallback for unknown metrics types
             logger.debug(
@@ -191,28 +203,27 @@ class MetricsLogObserver(BaseObserver):
             f"📊 {processor_info} LLM TOKEN USAGE{model_info}: {usage_str} at {time_sec:.2f}s"
         )
 
-    def _log_smart_turn(
+    def _log_turn(
         self,
-        metrics_data: SmartTurnMetricsData,
+        metrics_data: TurnMetricsData,
         processor_info: str,
         model_info: str,
         time_sec: float,
     ):
-        """Log smart turn prediction metrics.
+        """Log turn prediction metrics.
 
         Args:
-            metrics_data: The smart turn metrics data.
+            metrics_data: The turn metrics data.
             processor_info: Formatted processor name string.
             model_info: Formatted model name string.
             time_sec: Timestamp in seconds.
         """
         complete_str = "COMPLETE" if metrics_data.is_complete else "INCOMPLETE"
+        e2e_str = f"{metrics_data.e2e_processing_time_ms:.1f}ms"
 
         logger.debug(
-            f"📊 {processor_info} SMART TURN{model_info}: {complete_str} "
+            f"📊 {processor_info} TURN{model_info}: {complete_str} "
             f"(probability: {metrics_data.probability:.2%}, "
-            f"inference: {metrics_data.inference_time_ms:.1f}ms, "
-            f"server: {metrics_data.server_total_time_ms:.1f}ms, "
-            f"e2e: {metrics_data.e2e_processing_time_ms:.1f}ms) "
+            f"e2e: {e2e_str}) "
             f"at {time_sec:.2f}s"
         )

src/pipecat/observers/loggers/user_bot_latency_log_observer.py

@@ -4,9 +4,15 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-"""Observer for measuring user-to-bot response latency."""
+"""Observer for measuring user-to-bot response latency.
+
+.. deprecated:: 0.0.102
+    This module is deprecated. Use :class:`UserBotLatencyObserver` directly
+    with its ``on_latency_measured`` event handler instead.
+"""
 
 import time
+import warnings
 from statistics import mean
 
 from loguru import logger
@@ -27,6 +33,10 @@ class UserBotLatencyLogObserver(BaseObserver):
 
     This helps measure how quickly the AI services respond by tracking
     conversation turn timing and logging latency metrics.
+
+    .. deprecated:: 0.0.102
+        This class is deprecated. Use :class:`UserBotLatencyObserver` directly
+        with its ``on_latency_measured`` event handler for custom logging.
     """
 
     def __init__(self):
@@ -34,7 +44,17 @@ class UserBotLatencyLogObserver(BaseObserver):
 
         Sets up tracking for processed frames and user speech timing
         to calculate response latencies.
+
+        .. deprecated:: 0.0.102
+            This class is deprecated. Use :class:`UserBotLatencyObserver`
+            directly with its ``on_latency_measured`` event handler.
         """
+        warnings.warn(
+            "UserBotLatencyLogObserver is deprecated and will be removed in a future version. "
+            "Use UserBotLatencyObserver directly with its on_latency_measured event handler instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         super().__init__()
         self._user_bot_latency_processed_frames = set()
         self._user_stopped_time = 0
@@ -59,7 +79,7 @@ class UserBotLatencyLogObserver(BaseObserver):
         if isinstance(data.frame, VADUserStartedSpeakingFrame):
             self._user_stopped_time = 0
         elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
-            self._user_stopped_time = time.time()
+            self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
         elif isinstance(data.frame, (EndFrame, CancelFrame)):
             self._log_summary()
         elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time:

src/pipecat/observers/startup_timing_observer.py

@@ -0,0 +1,328 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Observer for tracking pipeline startup timing.
+
+This module provides an observer that measures how long each processor's
+``start()`` method takes during pipeline startup. It works by tracking
+when a ``StartFrame`` arrives at a processor (``on_process_frame``) versus
+when it leaves (``on_push_frame``), giving the exact ``start()`` duration
+for each processor in the pipeline.
+
+It also measures transport timing — the time from ``StartFrame`` to the
+first ``BotConnectedFrame`` (SFU transports only) and ``ClientConnectedFrame``
+— via a separate ``on_transport_timing_report`` event.
+
+Example::
+
+    observer = StartupTimingObserver()
+
+    @observer.event_handler("on_startup_timing_report")
+    async def on_report(observer, report):
+        for t in report.processor_timings:
+            print(f"{t.processor_name}: {t.duration_secs:.3f}s")
+
+    @observer.event_handler("on_transport_timing_report")
+    async def on_transport(observer, report):
+        if report.bot_connected_secs is not None:
+            print(f"Bot connected in {report.bot_connected_secs:.3f}s")
+        print(f"Client connected in {report.client_connected_secs:.3f}s")
+
+    task = PipelineTask(pipeline, observers=[observer])
+"""
+
+import time
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple, Type
+
+from pydantic import BaseModel, Field
+
+from pipecat.frames.frames import BotConnectedFrame, ClientConnectedFrame, StartFrame
+from pipecat.observers.base_observer import BaseObserver, FrameProcessed, FramePushed
+from pipecat.pipeline.base_pipeline import BasePipeline
+from pipecat.pipeline.pipeline import PipelineSource
+from pipecat.processors.frame_processor import FrameProcessor
+
+# Internal pipeline types excluded from tracking by default.
+_INTERNAL_TYPES = (PipelineSource, BasePipeline)
+
+
+@dataclass
+class _ArrivalInfo:
+    """Internal record of when a StartFrame arrived at a processor."""
+
+    processor: FrameProcessor
+    arrival_ts_ns: int
+
+
+class ProcessorStartupTiming(BaseModel):
+    """Startup timing for a single processor.
+
+    Parameters:
+        processor_name: The name of the processor.
+        start_offset_secs: Offset in seconds from the StartFrame to when this
+            processor's start() began.
+        duration_secs: How long the processor's start() took, in seconds.
+    """
+
+    processor_name: str
+    start_offset_secs: float
+    duration_secs: float
+
+
+class StartupTimingReport(BaseModel):
+    """Report of startup timings for all measured processors.
+
+    Parameters:
+        start_time: Unix timestamp when the first processor began starting.
+        total_duration_secs: Total wall-clock time from first to last processor start.
+        processor_timings: Per-processor timing data, in pipeline order.
+    """
+
+    start_time: float
+    total_duration_secs: float
+    processor_timings: List[ProcessorStartupTiming] = Field(default_factory=list)
+
+
+class TransportTimingReport(BaseModel):
+    """Time from pipeline start to transport connection milestones.
+
+    Parameters:
+        start_time: Unix timestamp of the StartFrame (pipeline start).
+        bot_connected_secs: Seconds from StartFrame to first BotConnectedFrame
+            (only set for SFU transports).
+        client_connected_secs: Seconds from StartFrame to first ClientConnectedFrame.
+    """
+
+    start_time: float
+    bot_connected_secs: Optional[float] = None
+    client_connected_secs: Optional[float] = None
+
+
+class StartupTimingObserver(BaseObserver):
+    """Observer that measures processor startup times during pipeline initialization.
+
+    Tracks how long each processor's ``start()`` method takes by measuring the
+    time between when a ``StartFrame`` arrives at a processor and when it is
+    pushed downstream. This captures WebSocket connections, API authentication,
+    model loading, and other initialization work.
+
+    Also measures transport timing, the time from ``StartFrame`` to connection
+    milestones:
+
+    - ``bot_connected_secs``: When the bot joins the transport room
+      (SFU transports only, triggered by ``BotConnectedFrame``).
+    - ``client_connected_secs``: When a remote participant connects
+      (triggered by ``ClientConnectedFrame``).
+
+    By default, internal pipeline processors (``PipelineSource``, ``Pipeline``)
+    are excluded from the report. Pass ``processor_types`` to measure only
+    specific types.
+
+    Event handlers available:
+
+    - on_startup_timing_report: Called once after startup completes with the full
+      timing report.
+    - on_transport_timing_report: Called once when the first client connects with a
+      TransportTimingReport containing client_connected_secs and bot_connected_secs
+      (if available).
+
+    Example::
+
+        observer = StartupTimingObserver(
+            processor_types=(STTService, TTSService)
+        )
+
+        @observer.event_handler("on_startup_timing_report")
+        async def on_report(observer, report):
+            for t in report.processor_timings:
+                logger.info(f"{t.processor_name}: {t.duration_secs:.3f}s")
+
+        @observer.event_handler("on_transport_timing_report")
+        async def on_transport(observer, report):
+            if report.bot_connected_secs is not None:
+                logger.info(f"Bot connected in {report.bot_connected_secs:.3f}s")
+            logger.info(f"Client connected in {report.client_connected_secs:.3f}s")
+
+        task = PipelineTask(pipeline, observers=[observer])
+
+    Args:
+        processor_types: Optional tuple of processor types to measure. If None,
+            all non-internal processors are measured.
+    """
+
+    def __init__(
+        self,
+        *,
+        processor_types: Optional[Tuple[Type[FrameProcessor], ...]] = None,
+        **kwargs,
+    ):
+        """Initialize the startup timing observer.
+
+        Args:
+            processor_types: Optional tuple of processor types to measure.
+                If None, all non-internal processors are measured.
+            **kwargs: Additional arguments passed to parent class.
+        """
+        super().__init__(**kwargs)
+        self._processor_types = processor_types
+
+        # Map processor ID -> arrival info.
+        self._arrivals: Dict[int, _ArrivalInfo] = {}
+
+        # Collected timings in pipeline order.
+        self._timings: List[ProcessorStartupTiming] = []
+
+        # Lock onto the first StartFrame we see (by frame ID).
+        self._start_frame_id: Optional[str] = None
+
+        # Whether we've already emitted the startup timing report.
+        self._startup_timing_reported = False
+
+        # Whether we've already measured transport timing.
+        self._transport_timing_reported = False
+
+        # Timestamp (ns) when we first see a StartFrame arrive at a processor.
+        self._start_frame_arrival_ns: Optional[int] = None
+
+        # Bot connected timing (stored for inclusion in the transport report).
+        self._bot_connected_secs: Optional[float] = None
+
+        # Wall clock time when the StartFrame was first seen.
+        self._start_wall_clock: Optional[float] = None
+
+        self._register_event_handler("on_startup_timing_report")
+        self._register_event_handler("on_transport_timing_report")
+
+    def _should_track(self, processor: FrameProcessor) -> bool:
+        """Check if a processor should be tracked for timing.
+
+        Args:
+            processor: The processor to check.
+
+        Returns:
+            True if the processor matches the filter or no filter is set.
+        """
+        if self._processor_types is not None:
+            return isinstance(processor, self._processor_types)
+        # Default: exclude internal pipeline plumbing.
+        return not isinstance(processor, _INTERNAL_TYPES)
+
+    async def on_pipeline_started(self):
+        """Emit the startup timing report when the pipeline has fully started.
+
+        Called by the ``PipelineTask`` after the ``StartFrame`` has been
+        processed by all processors, including nested ``ParallelPipeline``
+        branches.
+        """
+        if self._timings:
+            await self._emit_report()
+
+    async def on_process_frame(self, data: FrameProcessed):
+        """Record when a StartFrame arrives at a processor.
+
+        Args:
+            data: The frame processing event data.
+        """
+        if self._startup_timing_reported:
+            return
+
+        if not isinstance(data.frame, StartFrame):
+            return
+
+        # Lock onto the first StartFrame.
+        if self._start_frame_id is None:
+            self._start_frame_id = data.frame.id
+            self._start_frame_arrival_ns = data.timestamp
+            self._start_wall_clock = time.time()
+        elif data.frame.id != self._start_frame_id:
+            return
+
+        if self._should_track(data.processor):
+            self._arrivals[data.processor.id] = _ArrivalInfo(
+                processor=data.processor, arrival_ts_ns=data.timestamp
+            )
+
+    async def on_push_frame(self, data: FramePushed):
+        """Record when a StartFrame leaves a processor and compute the delta.
+
+        Also handles ``BotConnectedFrame`` and ``ClientConnectedFrame`` to
+        measure transport timing.
+
+        Args:
+            data: The frame push event data.
+        """
+        if isinstance(data.frame, BotConnectedFrame):
+            self._handle_bot_connected(data)
+            return
+
+        if isinstance(data.frame, ClientConnectedFrame):
+            await self._handle_client_connected(data)
+            return
+
+        if self._startup_timing_reported:
+            return
+
+        if not isinstance(data.frame, StartFrame):
+            return
+
+        if self._start_frame_id is not None and data.frame.id != self._start_frame_id:
+            return
+
+        arrival = self._arrivals.pop(data.source.id, None)
+        if arrival is None:
+            return
+
+        duration_ns = data.timestamp - arrival.arrival_ts_ns
+        duration_secs = duration_ns / 1e9
+        start_offset_secs = (arrival.arrival_ts_ns - self._start_frame_arrival_ns) / 1e9
+
+        self._timings.append(
+            ProcessorStartupTiming(
+                processor_name=arrival.processor.name,
+                start_offset_secs=start_offset_secs,
+                duration_secs=duration_secs,
+            )
+        )
+
+    def _handle_bot_connected(self, data: FramePushed):
+        """Record bot connected timing on first BotConnectedFrame."""
+        if self._bot_connected_secs is not None or self._start_frame_arrival_ns is None:
+            return
+
+        delta_ns = data.timestamp - self._start_frame_arrival_ns
+        self._bot_connected_secs = delta_ns / 1e9
+
+    async def _handle_client_connected(self, data: FramePushed):
+        """Emit transport timing report on first ClientConnectedFrame."""
+        if self._transport_timing_reported or self._start_frame_arrival_ns is None:
+            return
+
+        self._transport_timing_reported = True
+        delta_ns = data.timestamp - self._start_frame_arrival_ns
+        client_connected_secs = delta_ns / 1e9
+        report = TransportTimingReport(
+            start_time=self._start_wall_clock or 0.0,
+            bot_connected_secs=self._bot_connected_secs,
+            client_connected_secs=client_connected_secs,
+        )
+        await self._call_event_handler("on_transport_timing_report", report)
+
+    async def _emit_report(self):
+        """Build and emit the startup timing report."""
+        if self._startup_timing_reported:
+            return
+        self._startup_timing_reported = True
+
+        total = sum(t.duration_secs for t in self._timings)
+
+        report = StartupTimingReport(
+            start_time=self._start_wall_clock or 0.0,
+            total_duration_secs=total,
+            processor_timings=self._timings,
+        )
+
+        await self._call_event_handler("on_startup_timing_report", report)

src/pipecat/observers/user_bot_latency_observer.py

@@ -0,0 +1,351 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Observer for tracking user-to-bot response latency.
+
+This module provides an observer that monitors the time between when a user
+stops speaking and when the bot starts speaking, emitting events when latency
+is measured. Optionally collects per-service latency breakdown metrics
+(TTFB, text aggregation) when ``enable_metrics=True``.
+"""
+
+import time
+from collections import deque
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, Field
+
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    ClientConnectedFrame,
+    FunctionCallInProgressFrame,
+    FunctionCallResultFrame,
+    InterruptionFrame,
+    MetricsFrame,
+    UserStoppedSpeakingFrame,
+    VADUserStartedSpeakingFrame,
+    VADUserStoppedSpeakingFrame,
+)
+from pipecat.metrics.metrics import (
+    TextAggregationMetricsData,
+    TTFBMetricsData,
+)
+from pipecat.observers.base_observer import BaseObserver, FramePushed
+from pipecat.processors.frame_processor import FrameDirection
+
+
+class TTFBBreakdownMetrics(BaseModel):
+    """TTFB measurement with timestamp for timeline placement.
+
+    Parameters:
+        processor: Name of the processor that reported the TTFB.
+        model: Optional model name associated with the metric.
+        start_time: Unix timestamp when the TTFB measurement started.
+        duration_secs: TTFB duration in seconds.
+    """
+
+    processor: str
+    model: Optional[str] = None
+    start_time: float
+    duration_secs: float
+
+
+class TextAggregationBreakdownMetrics(BaseModel):
+    """Text aggregation measurement with timestamp for timeline placement.
+
+    Parameters:
+        processor: Name of the processor that reported the metric.
+        start_time: Unix timestamp when text aggregation started.
+        duration_secs: Aggregation duration in seconds.
+    """
+
+    processor: str
+    start_time: float
+    duration_secs: float
+
+
+class FunctionCallMetrics(BaseModel):
+    """Latency for a single function call execution.
+
+    Parameters:
+        function_name: Name of the function that was called.
+        start_time: Unix timestamp when execution started.
+        duration_secs: Time in seconds from execution start to result.
+    """
+
+    function_name: str
+    start_time: float
+    duration_secs: float
+
+
+class LatencyBreakdown(BaseModel):
+    """Per-service latency breakdown for a single user-to-bot cycle.
+
+    Collected between ``VADUserStoppedSpeakingFrame`` and
+    ``BotStartedSpeakingFrame`` when ``enable_metrics=True`` in
+    :class:`~pipecat.pipeline.task.PipelineParams`.
+
+    Parameters:
+        ttfb: Time-to-first-byte metrics from each service in the pipeline.
+        text_aggregation: First text aggregation measurement, representing
+            the latency cost of sentence aggregation in the TTS pipeline.
+        user_turn_start_time: Unix timestamp when the user turn started
+            (actual user silence, adjusted for VAD stop_secs). ``None`` if
+            no ``VADUserStoppedSpeakingFrame`` was observed.
+        user_turn_secs: Duration in seconds of the user's turn, measured
+            from when the user actually stopped speaking to when the turn
+            was released (``UserStoppedSpeakingFrame``). This includes
+            VAD silence detection, STT finalization, and any turn analyzer
+            wait. ``None`` if no ``UserStoppedSpeakingFrame`` was observed
+            (e.g. no turn analyzer configured).
+        function_calls: Latency for each function call executed during
+            this cycle. Empty if no function calls occurred.
+    """
+
+    ttfb: List[TTFBBreakdownMetrics] = Field(default_factory=list)
+    text_aggregation: Optional[TextAggregationBreakdownMetrics] = None
+    user_turn_start_time: Optional[float] = None
+    user_turn_secs: Optional[float] = None
+    function_calls: List[FunctionCallMetrics] = Field(default_factory=list)
+
+    def chronological_events(self) -> List[str]:
+        """Return human-readable event labels sorted by start time.
+
+        Collects all sub-metrics into a flat list, sorts by ``start_time``,
+        and returns formatted strings suitable for logging.
+
+        Returns:
+            List of formatted strings, one per event, in chronological order.
+        """
+        events: List[tuple] = []
+
+        if self.user_turn_start_time is not None and self.user_turn_secs is not None:
+            events.append((self.user_turn_start_time, f"User turn: {self.user_turn_secs:.3f}s"))
+
+        for t in self.ttfb:
+            events.append((t.start_time, f"{t.processor}: TTFB {t.duration_secs:.3f}s"))
+
+        for fc in self.function_calls:
+            events.append((fc.start_time, f"{fc.function_name}: {fc.duration_secs:.3f}s"))
+
+        if self.text_aggregation:
+            ta = self.text_aggregation
+            events.append(
+                (ta.start_time, f"{ta.processor}: text aggregation {ta.duration_secs:.3f}s")
+            )
+
+        events.sort(key=lambda e: e[0])
+        return [label for _, label in events]
+
+
+class UserBotLatencyObserver(BaseObserver):
+    """Observer that tracks user-to-bot response latency.
+
+    Measures the time between when a user stops speaking (VADUserStoppedSpeakingFrame)
+    and when the bot starts speaking (BotStartedSpeakingFrame). Emits events when
+    latency is measured, allowing consumers to log, trace, or otherwise process
+    the latency data.
+
+    When ``enable_metrics=True`` in pipeline params, also collects per-service
+    latency breakdown (TTFB, text aggregation) and emits an
+    ``on_latency_breakdown`` event alongside the existing latency measurement.
+
+    This observer follows the composition pattern used by TurnTrackingObserver,
+    acting as a reusable component for latency measurement.
+
+    Events:
+        on_latency_measured(observer, latency_seconds): Emitted when
+            time-to-first-bot-speech is calculated. Measures the time from
+            when the user stopped speaking to when the bot starts speaking.
+        on_latency_breakdown(observer, breakdown): Emitted at each
+            ``BotStartedSpeakingFrame`` with a :class:`LatencyBreakdown`
+            containing per-service metrics collected during the user→bot cycle.
+        on_first_bot_speech_latency(observer, latency_seconds): Emitted once,
+            the first time ``BotStartedSpeakingFrame`` arrives after
+            ``ClientConnectedFrame``. Measures the time from client connection
+            to the first bot speech.
+    """
+
+    def __init__(self, *, max_frames=100, **kwargs):
+        """Initialize the user-bot latency observer.
+
+        Sets up tracking for processed frames and user speech timing
+        to calculate response latencies.
+
+        Args:
+            max_frames: Maximum number of frame IDs to keep in history for
+                duplicate detection. Defaults to 100.
+            **kwargs: Additional arguments passed to parent class.
+        """
+        super().__init__(**kwargs)
+        self._user_stopped_time: Optional[float] = None
+        self._user_turn_start_time: Optional[float] = None
+        self._user_turn: Optional[float] = None
+
+        # First bot speech tracking
+        self._client_connected_time: Optional[float] = None
+        self._first_bot_speech_measured: bool = False
+
+        # Frame deduplication (bounded deque + set pattern)
+        self._processed_frames: set = set()
+        self._frame_history: deque = deque(maxlen=max_frames)
+
+        # Per-cycle metric accumulators
+        self._ttfb: List[TTFBBreakdownMetrics] = []
+        self._text_aggregation: Optional[TextAggregationBreakdownMetrics] = None
+        self._function_call_starts: Dict[str, tuple[str, float]] = {}
+        self._function_call_metrics: List[FunctionCallMetrics] = []
+
+        self._register_event_handler("on_latency_measured")
+        self._register_event_handler("on_latency_breakdown")
+        self._register_event_handler("on_first_bot_speech_latency")
+
+    async def on_push_frame(self, data: FramePushed):
+        """Process frames to track speech timing and calculate latency.
+
+        Tracks VAD events and bot speaking events to measure the time between
+        user stopping speech and bot starting speech. Also accumulates metrics
+        from MetricsFrame for the latency breakdown.
+
+        Args:
+            data: Frame push event containing the frame and direction information.
+        """
+        # Only process downstream frames
+        if data.direction != FrameDirection.DOWNSTREAM:
+            return
+
+        # Skip already processed frames (bounded deque + set)
+        if data.frame.id in self._processed_frames:
+            return
+
+        self._processed_frames.add(data.frame.id)
+        self._frame_history.append(data.frame.id)
+
+        if len(self._processed_frames) > len(self._frame_history):
+            self._processed_frames = set(self._frame_history)
+
+        # Track client connection (first occurrence only)
+        if isinstance(data.frame, ClientConnectedFrame):
+            if self._client_connected_time is None:
+                self._client_connected_time = time.time()
+            return
+
+        # Track speech and pipeline events for latency
+        if isinstance(data.frame, VADUserStartedSpeakingFrame):
+            # Reset when user starts speaking
+            self._user_stopped_time = None
+            self._user_turn_start_time = None
+            self._user_turn = None
+            self._reset_accumulators()
+            # If user speaks before the bot's first speech, abandon the
+            # first-bot-speech measurement — it's only meaningful for greetings.
+            self._first_bot_speech_measured = True
+        elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
+            # Record the actual time the user stopped speaking, which is
+            # the VAD determination time minus the stop_secs silence duration
+            # that had to elapse before the VAD confirmed speech ended.
+            self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
+            self._user_turn_start_time = self._user_stopped_time
+        elif isinstance(data.frame, UserStoppedSpeakingFrame):
+            # Measure the user turn duration: from actual user silence to
+            # turn release. Includes VAD silence detection, STT finalization,
+            # and any turn analyzer wait.
+            if self._user_stopped_time is not None:
+                self._user_turn = time.time() - self._user_stopped_time
+        elif isinstance(data.frame, InterruptionFrame):
+            # Discard stale metrics from cancelled LLM/TTS cycles
+            self._reset_accumulators()
+        elif isinstance(data.frame, FunctionCallInProgressFrame):
+            self._function_call_starts[data.frame.tool_call_id] = (
+                data.frame.function_name,
+                time.time(),
+            )
+        elif isinstance(data.frame, FunctionCallResultFrame):
+            start = self._function_call_starts.pop(data.frame.tool_call_id, None)
+            if start is not None:
+                function_name, start_time = start
+                self._function_call_metrics.append(
+                    FunctionCallMetrics(
+                        function_name=function_name,
+                        start_time=start_time,
+                        duration_secs=time.time() - start_time,
+                    )
+                )
+        elif isinstance(data.frame, MetricsFrame):
+            self._handle_metrics_frame(data.frame)
+        elif isinstance(data.frame, BotStartedSpeakingFrame):
+            await self._handle_bot_started_speaking()
+
+    async def _handle_bot_started_speaking(self):
+        """Handle BotStartedSpeakingFrame to emit latency and breakdown."""
+        emit_breakdown = False
+
+        # One-time first bot speech measurement (client connect → first speech)
+        if self._client_connected_time is not None and not self._first_bot_speech_measured:
+            self._first_bot_speech_measured = True
+            latency = time.time() - self._client_connected_time
+            await self._call_event_handler("on_first_bot_speech_latency", latency)
+            emit_breakdown = True
+
+        if self._user_stopped_time is not None:
+            latency = time.time() - self._user_stopped_time
+            self._user_stopped_time = None
+            await self._call_event_handler("on_latency_measured", latency)
+            emit_breakdown = True
+
+        if emit_breakdown:
+            breakdown = LatencyBreakdown(
+                ttfb=list(self._ttfb),
+                text_aggregation=self._text_aggregation,
+                user_turn_start_time=self._user_turn_start_time,
+                user_turn_secs=self._user_turn,
+                function_calls=list(self._function_call_metrics),
+            )
+            await self._call_event_handler("on_latency_breakdown", breakdown)
+            self._reset_accumulators()
+
+    def _handle_metrics_frame(self, frame: MetricsFrame):
+        """Extract latency metrics from a MetricsFrame.
+
+        Accumulates metrics when a measurement is in progress: either a
+        user→bot cycle (after ``VADUserStoppedSpeakingFrame``) or the
+        first-bot-speech window (after ``ClientConnectedFrame``).
+        """
+        waiting_for_first_speech = (
+            self._client_connected_time is not None and not self._first_bot_speech_measured
+        )
+        if self._user_stopped_time is None and not waiting_for_first_speech:
+            return
+
+        now = time.time()
+        for metrics_data in frame.data:
+            if isinstance(metrics_data, TTFBMetricsData) and metrics_data.value > 0:
+                self._ttfb.append(
+                    TTFBBreakdownMetrics(
+                        processor=metrics_data.processor,
+                        model=metrics_data.model,
+                        start_time=now - metrics_data.value,
+                        duration_secs=metrics_data.value,
+                    )
+                )
+            elif isinstance(metrics_data, TextAggregationMetricsData):
+                # Only keep the first measurement — it's the one that
+                # impacts the initial speaking latency.
+                if self._text_aggregation is None:
+                    self._text_aggregation = TextAggregationBreakdownMetrics(
+                        processor=metrics_data.processor,
+                        start_time=now - metrics_data.value,
+                        duration_secs=metrics_data.value,
+                    )
+
+    def _reset_accumulators(self):
+        """Clear per-cycle metric accumulators."""
+        self._ttfb = []
+        self._text_aggregation = None
+        self._user_turn_start_time = None
+        self._user_turn = None
+        self._function_call_starts = {}
+        self._function_call_metrics = []

src/pipecat/pipeline/llm_switcher.py

@@ -9,7 +9,11 @@
 from typing import Any, List, Optional, Type
 
 from pipecat.adapters.schemas.direct_function import DirectFunction
-from pipecat.pipeline.service_switcher import ServiceSwitcher, StrategyType
+from pipecat.pipeline.service_switcher import (
+    ServiceSwitcher,
+    ServiceSwitcherStrategyManual,
+    StrategyType,
+)
 from pipecat.processors.aggregators.llm_context import LLMContext
 from pipecat.services.llm_service import LLMService
 
@@ -19,18 +23,20 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
 
     Example::
 
-        llm_switcher = LLMSwitcher(
-            llms=[openai_llm, anthropic_llm],
-            strategy_type=ServiceSwitcherStrategyManual
-        )
+        llm_switcher = LLMSwitcher(llms=[openai_llm, anthropic_llm])
     """
 
-    def __init__(self, llms: List[LLMService], strategy_type: Type[StrategyType]):
+    def __init__(
+        self,
+        llms: List[LLMService],
+        strategy_type: Type[StrategyType] = ServiceSwitcherStrategyManual,
+    ):
         """Initialize the service switcher with a list of LLMs and a switching strategy.
 
         Args:
             llms: List of LLM services to switch between.
             strategy_type: The strategy class to use for switching between LLMs.
+                Defaults to ``ServiceSwitcherStrategyManual``.
         """
         super().__init__(llms, strategy_type)
 
@@ -44,7 +50,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
         return self.services
 
     @property
-    def active_llm(self) -> Optional[LLMService]:
+    def active_llm(self) -> LLMService:
         """Get the currently active LLM.
 
         Returns:
@@ -52,17 +58,19 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
         """
         return self.strategy.active_service
 
-    async def run_inference(self, context: LLMContext) -> Optional[str]:
+    async def run_inference(self, context: LLMContext, **kwargs) -> Optional[str]:
         """Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context, using the currently active LLM.
 
         Args:
             context: The LLM context containing conversation history.
+            **kwargs: Additional arguments forwarded to the active LLM's run_inference
+                (e.g. max_tokens, system_instruction).
 
         Returns:
             The LLM's response as a string, or None if no response is generated.
         """
         if self.active_llm:
-            return await self.active_llm.run_inference(context=context)
+            return await self.active_llm.run_inference(context=context, **kwargs)
         return None
 
     def register_function(
@@ -72,6 +80,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
         start_callback=None,
         *,
         cancel_on_interruption: bool = True,
+        timeout_secs: Optional[float] = None,
     ):
         """Register a function handler for LLM function calls, on all LLMs, active or not.
 
@@ -88,6 +97,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
 
             cancel_on_interruption: Whether to cancel this function call when an
                 interruption occurs. Defaults to True.
+            timeout_secs: Optional timeout in seconds for the function call.
         """
         for llm in self.llms:
             llm.register_function(
@@ -95,6 +105,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
                 handler=handler,
                 start_callback=start_callback,
                 cancel_on_interruption=cancel_on_interruption,
+                timeout_secs=timeout_secs,
             )
 
     def register_direct_function(
@@ -102,6 +113,7 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
         handler: DirectFunction,
         *,
         cancel_on_interruption: bool = True,
+        timeout_secs: Optional[float] = None,
     ):
         """Register a direct function handler for LLM function calls, on all LLMs, active or not.
 
@@ -109,9 +121,11 @@ class LLMSwitcher(ServiceSwitcher[StrategyType]):
             handler: The direct function to register. Must follow DirectFunction protocol.
             cancel_on_interruption: Whether to cancel this function call when an
                 interruption occurs. Defaults to True.
+            timeout_secs: Optional timeout in seconds for the function call.
         """
         for llm in self.llms:
             llm.register_direct_function(
                 handler=handler,
                 cancel_on_interruption=cancel_on_interruption,
+                timeout_secs=timeout_secs,
             )

src/pipecat/pipeline/parallel_pipeline.py

@@ -52,6 +52,8 @@ class ParallelPipeline(BasePipeline):
 
         self._seen_ids = set()
         self._frame_counter: Dict[int, int] = {}
+        self._synchronizing: bool = False
+        self._buffered_frames: list[tuple[Frame, FrameDirection]] = []
 
         logger.debug(f"Creating {self} pipelines")
 
@@ -141,8 +143,22 @@ class ParallelPipeline(BasePipeline):
         await super().process_frame(frame, direction)
 
         # Parallel pipeline synchronized frames.
+        #
+        # - StartFrame: If a fast branch completes first, processors in
+        #   other branches that haven't received StartFrame yet could
+        #   receive other frames before it, causing errors.
+        #
+        # - EndFrame: If EndFrame escapes from a fast branch, downstream
+        #   processors (e.g. output transport) begin shutting down while
+        #   other branches still have frames to flush, causing lost output.
+        #
+        # - CancelFrame: PipelineTask waits for CancelFrame to reach the
+        #   pipeline sink. If it escapes from a fast branch while slower
+        #   branches are still running, the task considers cancellation
+        #   complete prematurely.
         if isinstance(frame, (StartFrame, EndFrame, CancelFrame)):
             self._frame_counter[frame.id] = len(self._pipelines)
+            self._synchronizing = True
             await self.pause_processing_system_frames()
             await self.pause_processing_frames()
 
@@ -151,10 +167,18 @@ class ParallelPipeline(BasePipeline):
             await p.queue_frame(frame, direction)
 
     async def _parallel_push_frame(self, frame: Frame, direction: FrameDirection):
-        """Push frames while avoiding duplicates using frame ID tracking."""
+        """Push frames while avoiding duplicates using frame ID tracking.
+
+        During lifecycle frame synchronization, non-lifecycle frames are buffered
+        to prevent them from escaping the parallel pipeline before all branches
+        have finished processing the lifecycle frame.
+        """
         if frame.id not in self._seen_ids:
             self._seen_ids.add(frame.id)
-            await self.push_frame(frame, direction)
+            if self._synchronizing:
+                self._buffered_frames.append((frame, direction))
+            else:
+                await self.push_frame(frame, direction)
 
     async def _pipeline_sink_push_frame(self, frame: Frame, direction: FrameDirection):
         # Parallel pipeline synchronized frames.
@@ -167,8 +191,21 @@ class ParallelPipeline(BasePipeline):
 
             # Only push the frame when all pipelines have processed it.
             if frame_counter == 0:
-                await self._parallel_push_frame(frame, direction)
+                self._synchronizing = False
+                # StartFrame should always go before any other frame.
+                if isinstance(frame, StartFrame):
+                    await self._parallel_push_frame(frame, direction)
+                    await self._flush_buffered_frames()
+                else:
+                    await self._flush_buffered_frames()
+                    await self._parallel_push_frame(frame, direction)
                 await self.resume_processing_system_frames()
                 await self.resume_processing_frames()
         else:
             await self._parallel_push_frame(frame, direction)
+
+    async def _flush_buffered_frames(self):
+        """Flush frames that were buffered during lifecycle frame synchronization."""
+        while len(self._buffered_frames) > 0:
+            frame, direction = self._buffered_frames.pop(0)
+            await self.push_frame(frame, direction)

src/pipecat/pipeline/service_switcher.py

@@ -6,26 +6,40 @@
 
 """Service switcher for switching between different services at runtime, with different switching strategies."""
 
-from dataclasses import dataclass
 from typing import Any, Generic, List, Optional, Type, TypeVar
 
+from loguru import logger
+
 from pipecat.frames.frames import (
-    ControlFrame,
+    ErrorFrame,
     Frame,
     ManuallySwitchServiceFrame,
+    ServiceMetadataFrame,
     ServiceSwitcherFrame,
+    ServiceSwitcherRequestMetadataFrame,
 )
 from pipecat.pipeline.parallel_pipeline import ParallelPipeline
 from pipecat.processors.filters.function_filter import FunctionFilter
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.utils.base_object import BaseObject
 
 
-class ServiceSwitcherStrategy:
+class ServiceSwitcherStrategy(BaseObject):
     """Base class for service switching strategies.
 
     Note:
         Strategy classes are instantiated internally by ServiceSwitcher.
         Developers should pass the strategy class (not an instance) to ServiceSwitcher.
+
+    Event handlers available:
+
+    - on_service_switched: Called when the active service changes.
+
+    Example::
+
+        @strategy.event_handler("on_service_switched")
+        async def on_service_switched(strategy, service):
+            ...
     """
 
     def __init__(self, services: List[FrameProcessor]):
@@ -37,20 +51,76 @@ class ServiceSwitcherStrategy:
         Args:
             services: List of frame processors to switch between.
         """
-        self.services = services
-        self.active_service: Optional[FrameProcessor] = None
+        super().__init__()
 
-    def handle_frame(self, frame: ServiceSwitcherFrame, direction: FrameDirection):
+        if len(services) == 0:
+            raise Exception(f"ServiceSwitcherStrategy needs at least one service")
+
+        self._services = services
+        self._active_service = services[0]
+
+        self._register_event_handler("on_service_switched")
+
+    @property
+    def services(self) -> List[FrameProcessor]:
+        """Return the list of available services."""
+        return self._services
+
+    @property
+    def active_service(self) -> FrameProcessor:
+        """Return the currently active service."""
+        return self._active_service
+
+    async def handle_frame(
+        self, frame: ServiceSwitcherFrame, direction: FrameDirection
+    ) -> Optional[FrameProcessor]:
         """Handle a frame that controls service switching.
 
-        This method can be overridden by subclasses to implement specific logic
-        for handling frames that control service switching.
+        The base implementation returns ``None`` for all frames. Subclasses
+        override this to implement specific switching behaviors.
 
         Args:
             frame: The frame to handle.
             direction: The direction of the frame (upstream or downstream).
+
+        Returns:
+            The newly active service if a switch occurred, or None otherwise.
         """
-        raise NotImplementedError("Subclasses must implement this method.")
+        return None
+
+    async def handle_error(self, error: ErrorFrame) -> Optional[FrameProcessor]:
+        """Handle an error from the active service.
+
+        Called by ``ServiceSwitcher`` when a non-fatal ``ErrorFrame`` is pushed
+        upstream by the currently active service. Subclasses can override this
+        to implement automatic failover.
+
+        Args:
+            error: The error frame pushed by the active service.
+
+        Returns:
+            The newly active service if a switch occurred, or None otherwise.
+        """
+        return None
+
+    async def _set_active_if_available(self, service: FrameProcessor) -> Optional[FrameProcessor]:
+        """Set the active service to the given one, if it is in the list of available services.
+
+        If it's not in the list, the request is ignored, as it may have been
+        intended for another ServiceSwitcher in the pipeline.
+
+        Args:
+            service: The service to set as active.
+
+        Returns:
+            The newly active service, or None if the service was not found.
+        """
+        if service in self.services:
+            self._active_service = service
+            await service.queue_frame(ServiceSwitcherRequestMetadataFrame(service=service))
+            await self._call_event_handler("on_service_switched", service)
+            return service
+        return None
 
 
 class ServiceSwitcherStrategyManual(ServiceSwitcherStrategy):
@@ -67,103 +137,115 @@ class ServiceSwitcherStrategyManual(ServiceSwitcherStrategy):
         )
     """
 
-    def __init__(self, services: List[FrameProcessor]):
-        """Initialize the manual service switcher strategy with a list of services.
-
-        Note:
-            This is called internally by ServiceSwitcher. Do not instantiate directly.
-
-        Args:
-            services: List of frame processors to switch between.
-        """
-        super().__init__(services)
-        self.active_service = services[0] if services else None
-
-    def handle_frame(self, frame: ServiceSwitcherFrame, direction: FrameDirection):
+    async def handle_frame(
+        self, frame: ServiceSwitcherFrame, direction: FrameDirection
+    ) -> Optional[FrameProcessor]:
         """Handle a frame that controls service switching.
 
         Args:
             frame: The frame to handle.
             direction: The direction of the frame (upstream or downstream).
+
+        Returns:
+            The newly active service if a switch occurred, or None otherwise.
         """
         if isinstance(frame, ManuallySwitchServiceFrame):
-            self._set_active_if_available(frame.service)
-        else:
-            raise ValueError(f"Unsupported frame type: {type(frame)}")
+            return await self._set_active_if_available(frame.service)
 
-    def _set_active_if_available(self, service: FrameProcessor):
-        """Set the active service to the given one, if it is in the list of available services.
+        return None
 
-        If it's not in the list, the request is ignored, as it may have been
-        intended for another ServiceSwitcher in the pipeline.
+
+class ServiceSwitcherStrategyFailover(ServiceSwitcherStrategyManual):
+    """A strategy that automatically switches to a backup service on failure.
+
+    When the active service produces a non-fatal error, this strategy switches
+    to the next available service in the list. Recovery and fallback policies
+    are left to application code via the ``on_service_switched`` event.
+
+    Event handlers available:
+
+    - on_service_switched: Called when the active service changes.
+
+    Example::
+
+        switcher = ServiceSwitcher(
+            services=[primary_stt, backup_stt],
+            strategy_type=ServiceSwitcherStrategyFailover,
+        )
+
+        @switcher.strategy.event_handler("on_service_switched")
+        async def on_switched(strategy, service):
+            # App decides when/how to recover the failed service
+            ...
+    """
+
+    async def handle_error(self, error: ErrorFrame) -> Optional[FrameProcessor]:
+        """Handle an error from the active service by failing over.
+
+        Switches to the next service in the list. The failed service remains
+        in the list and can be switched back to manually or via application
+        logic in the ``on_service_switched`` event handler.
 
         Args:
-            service: The service to set as active.
+            error: The error frame pushed by the active service.
+
+        Returns:
+            The newly active service if a switch occurred, or None if no
+            other service is available.
         """
-        if service in self.services:
-            self.active_service = service
+        logger.warning(f"Service {self._active_service.name} reported an error: {error.error}")
+
+        if len(self._services) <= 1:
+            logger.error("No other service available to switch to")
+            return None
+
+        current_idx = self._services.index(self._active_service)
+        next_idx = (current_idx + 1) % len(self._services)
+        return await self._set_active_if_available(self._services[next_idx])
 
 
 StrategyType = TypeVar("StrategyType", bound=ServiceSwitcherStrategy)
 
 
 class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]):
-    """A pipeline that switches between different services at runtime."""
+    """Parallel pipeline that routes frames to one active service at a time.
 
-    def __init__(self, services: List[FrameProcessor], strategy_type: Type[StrategyType]):
+    Wraps each service in a pair of filters that gate frame flow based on
+    which service is currently active. Switching is controlled by
+    `ServiceSwitcherFrame` frames and delegated to a pluggable
+    `ServiceSwitcherStrategy`.
+
+    Example::
+
+        switcher = ServiceSwitcher(services=[stt_1, stt_2])
+    """
+
+    def __init__(
+        self,
+        services: List[FrameProcessor],
+        strategy_type: Type[StrategyType] = ServiceSwitcherStrategyManual,
+    ):
         """Initialize the service switcher with a list of services and a switching strategy.
 
         Args:
             services: List of frame processors to switch between.
             strategy_type: The strategy class to use for switching between services.
+                Defaults to ``ServiceSwitcherStrategyManual``.
         """
-        strategy = strategy_type(services)
-        super().__init__(*self._make_pipeline_definitions(services, strategy))
-        self.services = services
-        self.strategy = strategy
+        _strategy = strategy_type(services)
+        super().__init__(*self._make_pipeline_definitions(services, _strategy))
+        self._services = services
+        self._strategy = _strategy
 
-    class ServiceSwitcherFilter(FunctionFilter):
-        """An internal filter that allows frames to pass through to the wrapped service only if it's the active service."""
+    @property
+    def strategy(self) -> StrategyType:
+        """Return the active switching strategy."""
+        return self._strategy
 
-        def __init__(
-            self,
-            wrapped_service: FrameProcessor,
-            active_service: FrameProcessor,
-            direction: FrameDirection,
-        ):
-            """Initialize the service switcher filter with a strategy and direction.
-
-            Args:
-                wrapped_service: The service that this filter wraps.
-                active_service: The currently active service.
-                direction: The direction of frame flow to filter.
-            """
-            self._wrapped_service = wrapped_service
-            self._active_service = active_service
-
-            async def filter(_: Frame) -> bool:
-                return self._wrapped_service == self._active_service
-
-            super().__init__(filter, direction, filter_system_frames=True)
-
-        async def process_frame(self, frame, direction):
-            """Process a frame through the filter, handling special internal filter-updating frames."""
-            if isinstance(frame, ServiceSwitcher.ServiceSwitcherFilterFrame):
-                self._active_service = frame.active_service
-                # Two ServiceSwitcherFilters "sandwich" a service. Push the
-                # frame only to update the other side of the sandwich, but
-                # otherwise don't let it leave the sandwich.
-                if direction == self._direction:
-                    await self.push_frame(frame, direction)
-                return
-
-            await super().process_frame(frame, direction)
-
-    @dataclass
-    class ServiceSwitcherFilterFrame(ControlFrame):
-        """An internal frame used by ServiceSwitcher to filter frames based on active service."""
-
-        active_service: FrameProcessor
+    @property
+    def services(self) -> List[FrameProcessor]:
+        """Return the list of available services."""
+        return self._services
 
     @staticmethod
     def _make_pipeline_definitions(
@@ -178,20 +260,64 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]):
     def _make_pipeline_definition(
         service: FrameProcessor, strategy: ServiceSwitcherStrategy
     ) -> Any:
+        async def filter(_: Frame) -> bool:
+            return service == strategy.active_service
+
+        # Layout: Filter → Service → Filter
+        #
+        # filter_system_frames: we want to run filter functions also on system
+        # frames.
+        #
+        # enable_direct_mode: filter functions are quick so we don't need
+        # additional tasks.
         return [
-            ServiceSwitcher.ServiceSwitcherFilter(
-                wrapped_service=service,
-                active_service=strategy.active_service,
+            FunctionFilter(
+                filter=filter,
                 direction=FrameDirection.DOWNSTREAM,
+                filter_system_frames=True,
+                enable_direct_mode=True,
             ),
             service,
-            ServiceSwitcher.ServiceSwitcherFilter(
-                wrapped_service=service,
-                active_service=strategy.active_service,
+            FunctionFilter(
+                filter=filter,
                 direction=FrameDirection.UPSTREAM,
+                filter_system_frames=True,
+                enable_direct_mode=True,
             ),
         ]
 
+    async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
+        """Push a frame out of the service switcher.
+
+        Suppresses `ServiceSwitcherRequestMetadataFrame` targeting the active
+        service (since it has already been handled) and `ServiceMetadataFrame`
+        from inactive services so only the active service's metadata reaches
+        downstream processors. One case this happens is with `StartFrame` since
+        all the filters let it pass, and `StartFrame` causes the service to
+        generate `ServiceMetadataFrame`.
+
+        Non-fatal ``ErrorFrame`` instances are forwarded to the strategy via
+        ``handle_error`` so strategies like ``ServiceSwitcherStrategyFailover``
+        can perform failover. The error frame is still propagated upstream so
+        that application-level error handlers can observe it.
+        """
+        # Consume ServiceSwitcherRequestMetadataFrame once the targeted service
+        # has handled it (i.e. the active service).
+        if isinstance(frame, ServiceSwitcherRequestMetadataFrame):
+            if frame.service == self.strategy.active_service:
+                return
+
+        # Only let metadata from the active service escape.
+        if isinstance(frame, ServiceMetadataFrame):
+            if frame.service_name != self.strategy.active_service.name:
+                return
+
+        # Let the strategy react to non-fatal errors from the active service.
+        if isinstance(frame, ErrorFrame) and not frame.fatal:
+            await self.strategy.handle_error(frame)
+
+        await super().push_frame(frame, direction)
+
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         """Process a frame, handling frames which affect service switching.
 
@@ -199,11 +325,12 @@ class ServiceSwitcher(ParallelPipeline, Generic[StrategyType]):
             frame: The frame to process.
             direction: The direction of the frame (upstream or downstream).
         """
-        await super().process_frame(frame, direction)
-
         if isinstance(frame, ServiceSwitcherFrame):
-            self.strategy.handle_frame(frame, direction)
-            service_switcher_filter_frame = ServiceSwitcher.ServiceSwitcherFilterFrame(
-                active_service=self.strategy.active_service
-            )
-            await super().process_frame(service_switcher_filter_frame, direction)
+            service = await self.strategy.handle_frame(frame, direction)
+
+            # If we don't switch to a new service we need to keep processing the
+            # frame. If we switched, we just swallow the frame.
+            if not service:
+                await super().process_frame(frame, direction)
+        else:
+            await super().process_frame(frame, direction)

src/pipecat/pipeline/sync_parallel_pipeline.py

@@ -4,15 +4,21 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-"""Synchronous parallel pipeline implementation for concurrent frame processing.
+"""Synchronized parallel pipeline that holds output until all branches finish.
 
-This module provides a pipeline that processes frames through multiple parallel
-pipelines simultaneously, synchronizing their output to maintain frame ordering
-and prevent duplicate processing.
+A SyncParallelPipeline fans each inbound frame out to multiple parallel pipelines
+and waits for every pipeline to finish processing before releasing any of the
+resulting output frames. This ensures that all frames produced in response to a
+single input frame are emitted together.
+
+System frames (except EndFrame) are exempt from this synchronization — they pass
+straight through without waiting, since they are expected to race ahead of
+regular data frames.
 """
 
 import asyncio
 from dataclasses import dataclass
+from enum import Enum
 from itertools import chain
 from typing import List
 
@@ -24,22 +30,42 @@ from pipecat.pipeline.pipeline import Pipeline
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
 
 
+class FrameOrder(Enum):
+    """Controls the order in which synchronized frames are pushed downstream.
+
+    When multiple parallel pipelines produce output for the same input frame,
+    this setting determines the order in which those output frames are pushed.
+
+    Attributes:
+        ARRIVAL: Frames are pushed in the order they arrive from any pipeline.
+            This is the default and matches the behavior of prior versions.
+        PIPELINE: Frames are pushed in pipeline definition order — all frames
+            from the first pipeline are pushed, then all frames from the second
+            pipeline, and so on. Useful when the relative ordering between
+            pipelines matters (e.g. ensuring image frames precede audio frames).
+    """
+
+    ARRIVAL = "arrival"
+    PIPELINE = "pipeline"
+
+
 @dataclass
 class SyncFrame(ControlFrame):
-    """Control frame used to synchronize parallel pipeline processing.
+    """Sentinel frame used to detect when a parallel pipeline has finished processing.
 
-    This frame is sent through parallel pipelines to determine when the
-    internal pipelines have finished processing a batch of frames.
+    After sending a real frame into a parallel pipeline, a SyncFrame is sent
+    behind it. When the SyncFrame emerges from the pipeline's output, we know
+    all output frames for the preceding input have been produced.
     """
 
     pass
 
 
 class SyncParallelPipelineSource(FrameProcessor):
-    """Source processor for synchronous parallel pipeline processing.
+    """Bookend processor placed at the start of each parallel pipeline.
 
-    Routes frames to parallel pipelines and collects upstream responses
-    for synchronization purposes.
+    Forwards downstream frames into the pipeline and captures upstream frames
+    into a queue so the parent SyncParallelPipeline can release them later.
     """
 
     def __init__(self, upstream_queue: asyncio.Queue):
@@ -68,10 +94,11 @@ class SyncParallelPipelineSource(FrameProcessor):
 
 
 class SyncParallelPipelineSink(FrameProcessor):
-    """Sink processor for synchronous parallel pipeline processing.
+    """Bookend processor placed at the end of each parallel pipeline.
 
-    Collects downstream frames from parallel pipelines and routes
-    upstream frames back through the pipeline.
+    Captures downstream output frames into a queue so the parent
+    SyncParallelPipeline can release them later, and forwards upstream
+    frames back through the pipeline.
     """
 
     def __init__(self, downstream_queue: asyncio.Queue):
@@ -100,29 +127,44 @@ class SyncParallelPipelineSink(FrameProcessor):
 
 
 class SyncParallelPipeline(BasePipeline):
-    """Pipeline that processes frames through multiple parallel pipelines synchronously.
+    """Fans each input frame to parallel pipelines then holds output until every pipeline finishes.
 
-    Creates multiple parallel processing paths that all receive the same input frames
-    and produces synchronized output. Each parallel path is a separate pipeline that
-    processes frames independently, with synchronization points to ensure consistent
-    ordering and prevent duplicate frame processing.
+    For each inbound frame the pipeline:
 
-    The pipeline uses SyncFrame control frames to coordinate between parallel paths
-    and ensure all paths have completed processing before moving to the next frame.
+    1. Sends the frame into every parallel pipeline.
+    2. Sends a ``SyncFrame`` sentinel behind it in each pipeline.
+    3. Waits until every pipeline has produced its ``SyncFrame``, meaning all
+       output for that input is ready.
+    4. Releases the collected output frames (deduplicating by frame id, since
+       the same frame may emerge from more than one branch).
+
+    System frames (except ``EndFrame``) bypass this mechanism entirely — they are
+    forwarded through each pipeline and pushed immediately, since system frames
+    are expected to race ahead of regular data frames.
+
+    By default, output frames are pushed in the order they arrive from any pipeline
+    (``FrameOrder.ARRIVAL``). Set ``frame_order=FrameOrder.PIPELINE`` to push frames
+    in pipeline definition order instead — all output from the first pipeline, then
+    the second, and so on.
     """
 
-    def __init__(self, *args):
+    def __init__(self, *args, frame_order: FrameOrder = FrameOrder.ARRIVAL):
         """Initialize the synchronous parallel pipeline.
 
         Args:
-            *args: Variable number of processor lists, each representing a parallel pipeline path.
-                   Each argument should be a list of FrameProcessor instances.
+            *args: Variable number of processor lists, each representing a parallel
+                pipeline path. Each argument should be a list of FrameProcessor instances.
+            frame_order: Controls the order in which synchronized output frames are
+                pushed. ``FrameOrder.ARRIVAL`` (default) pushes frames in the order they arrive.
+                ``FrameOrder.PIPELINE`` pushes all frames from the first pipeline
+                before the second, and so on.
 
         Raises:
             Exception: If no arguments are provided.
             TypeError: If any argument is not a list of processors.
         """
         super().__init__()
+        self._frame_order = frame_order
 
         if len(args) == 0:
             raise Exception(f"SyncParallelPipeline needs at least one argument")
@@ -184,7 +226,7 @@ class SyncParallelPipeline(BasePipeline):
         Returns:
             The list of entry processors.
         """
-        return self._sources
+        return [s["processor"] for s in self._sources]
 
     def processors_with_metrics(self) -> List[FrameProcessor]:
         """Collect processors that can generate metrics from all parallel pipelines.
@@ -209,11 +251,11 @@ class SyncParallelPipeline(BasePipeline):
         await asyncio.gather(*[p.cleanup() for p in self._pipelines])
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
-        """Process frames through all parallel pipelines with synchronization.
+        """Send a frame through all parallel pipelines and release output once all finish.
 
-        Distributes frames to all parallel pipelines and synchronizes their output
-        to maintain proper ordering and prevent duplicate processing. Uses SyncFrame
-        control frames to coordinate between parallel paths.
+        System frames (except EndFrame) skip synchronization and pass straight
+        through. All other frames are fanned out to every pipeline, and output is
+        held until every pipeline signals completion (via SyncFrame).
 
         Args:
             frame: The frame to process.
@@ -221,60 +263,102 @@ class SyncParallelPipeline(BasePipeline):
         """
         await super().process_frame(frame, direction)
 
+        # SystemFrames (but not EndFrame) are simply passed through all
+        # internal pipelines without draining queued output. This avoids
+        # the race condition where a SystemFrame's wait_for_sync steals
+        # frames from a concurrent non-SystemFrame's wait_for_sync.
+        if isinstance(frame, SystemFrame) and not isinstance(frame, EndFrame):
+            if direction == FrameDirection.UPSTREAM:
+                for s in self._sinks:
+                    await s["processor"].process_frame(frame, direction)
+            elif direction == FrameDirection.DOWNSTREAM:
+                for s in self._sources:
+                    await s["processor"].process_frame(frame, direction)
+            await self.push_frame(frame, direction)
+            return
+
+        use_pipeline_order = self._frame_order == FrameOrder.PIPELINE
+
         # The last processor of each pipeline needs to be synchronous otherwise
-        # this element won't work. Since, we know it should be synchronous we
+        # this element won't work. Since we know it should be synchronous we
         # push a SyncFrame. Since frames are ordered we know this frame will be
         # pushed after the synchronous processor has pushed its data allowing us
-        # to synchrnonize all the internal pipelines by waiting for the
+        # to synchronize all the internal pipelines by waiting for the
         # SyncFrame in all of them.
+        #
+        # In ARRIVAL mode, output frames are put onto a shared main_queue as
+        # they arrive. In PIPELINE mode, they are accumulated in a per-pipeline
+        # list and returned so the caller can drain them in definition order.
         async def wait_for_sync(
             obj, main_queue: asyncio.Queue, frame: Frame, direction: FrameDirection
-        ):
+        ) -> list[Frame]:
             processor = obj["processor"]
             queue = obj["queue"]
+            output_frames: list[Frame] = []
 
             await processor.process_frame(frame, direction)
 
-            if isinstance(frame, (SystemFrame, EndFrame)):
+            if isinstance(frame, EndFrame):
                 new_frame = await queue.get()
-                if isinstance(new_frame, (SystemFrame, EndFrame)):
-                    await main_queue.put(new_frame)
-                else:
-                    while not isinstance(new_frame, (SystemFrame, EndFrame)):
+                if isinstance(new_frame, EndFrame):
+                    if use_pipeline_order:
+                        output_frames.append(new_frame)
+                    else:
                         await main_queue.put(new_frame)
+                else:
+                    while not isinstance(new_frame, EndFrame):
+                        if use_pipeline_order:
+                            output_frames.append(new_frame)
+                        else:
+                            await main_queue.put(new_frame)
                         queue.task_done()
                         new_frame = await queue.get()
             else:
                 await processor.process_frame(SyncFrame(), direction)
                 new_frame = await queue.get()
                 while not isinstance(new_frame, SyncFrame):
-                    await main_queue.put(new_frame)
+                    if use_pipeline_order:
+                        output_frames.append(new_frame)
+                    else:
+                        await main_queue.put(new_frame)
                     queue.task_done()
                     new_frame = await queue.get()
 
+            return output_frames
+
         if direction == FrameDirection.UPSTREAM:
             # If we get an upstream frame we process it in each sink.
-            await asyncio.gather(
+            frames_per_pipeline = await asyncio.gather(
                 *[wait_for_sync(s, self._up_queue, frame, direction) for s in self._sinks]
             )
         elif direction == FrameDirection.DOWNSTREAM:
             # If we get a downstream frame we process it in each source.
-            await asyncio.gather(
+            frames_per_pipeline = await asyncio.gather(
                 *[wait_for_sync(s, self._down_queue, frame, direction) for s in self._sources]
             )
 
-        seen_ids = set()
-        while not self._up_queue.empty():
-            frame = await self._up_queue.get()
-            if frame.id not in seen_ids:
-                await self.push_frame(frame, FrameDirection.UPSTREAM)
-                seen_ids.add(frame.id)
-            self._up_queue.task_done()
+        if use_pipeline_order:
+            # Push frames in pipeline definition order, deduplicating by id.
+            seen_ids = set()
+            for pipeline_frames in frames_per_pipeline:
+                for f in pipeline_frames:
+                    if f.id not in seen_ids:
+                        await self.push_frame(f, direction)
+                        seen_ids.add(f.id)
+        else:
+            # ARRIVAL mode: drain the shared queues in the order frames arrived.
+            seen_ids = set()
+            while not self._up_queue.empty():
+                frame = await self._up_queue.get()
+                if frame.id not in seen_ids:
+                    await self.push_frame(frame, FrameDirection.UPSTREAM)
+                    seen_ids.add(frame.id)
+                self._up_queue.task_done()
 
-        seen_ids = set()
-        while not self._down_queue.empty():
-            frame = await self._down_queue.get()
-            if frame.id not in seen_ids:
-                await self.push_frame(frame, FrameDirection.DOWNSTREAM)
-                seen_ids.add(frame.id)
-            self._down_queue.task_done()
+            seen_ids = set()
+            while not self._down_queue.empty():
+                frame = await self._down_queue.get()
+                if frame.id not in seen_ids:
+                    await self.push_frame(frame, FrameDirection.DOWNSTREAM)
+                    seen_ids.add(frame.id)
+                self._down_queue.task_done()

src/pipecat/pipeline/task.py

@@ -15,7 +15,7 @@ import asyncio
 import importlib.util
 import os
 from pathlib import Path
-from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Tuple, Type
+from typing import Any, AsyncIterable, Dict, Iterable, List, Optional, Set, Tuple, Type, TypeVar
 
 from loguru import logger
 from pydantic import BaseModel, ConfigDict, Field
@@ -43,14 +43,17 @@ from pipecat.frames.frames import (
 from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
 from pipecat.observers.base_observer import BaseObserver, FramePushed
 from pipecat.observers.turn_tracking_observer import TurnTrackingObserver
+from pipecat.observers.user_bot_latency_observer import UserBotLatencyObserver
 from pipecat.pipeline.base_pipeline import BasePipeline
 from pipecat.pipeline.base_task import BasePipelineTask, PipelineTaskParams
 from pipecat.pipeline.pipeline import Pipeline, PipelineSink, PipelineSource
 from pipecat.pipeline.task_observer import TaskObserver
 from pipecat.processors.aggregators.llm_response import LLMUserContextAggregator
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup
+from pipecat.processors.frameworks.rtvi import RTVIObserver, RTVIObserverParams, RTVIProcessor
 from pipecat.utils.asyncio.task_manager import BaseTaskManager, TaskManager, TaskManagerParams
 from pipecat.utils.tracing.setup import is_tracing_available
+from pipecat.utils.tracing.tracing_context import TracingContext
 from pipecat.utils.tracing.turn_trace_observer import TurnTraceObserver
 
 HEARTBEAT_SECS = 1.0
@@ -61,6 +64,9 @@ IDLE_TIMEOUT_SECS = 300
 CANCEL_TIMEOUT_SECS = 20.0
 
 
+T = TypeVar("T")
+
+
 class IdleFrameObserver(BaseObserver):
     """Idle timeout observer.
 
@@ -225,9 +231,12 @@ class PipelineTask(BasePipelineTask):
         conversation_id: Optional[str] = None,
         enable_tracing: bool = False,
         enable_turn_tracking: bool = True,
+        enable_rtvi: bool = True,
         idle_timeout_frames: Tuple[Type[Frame], ...] = (BotSpeakingFrame, UserSpeakingFrame),
         idle_timeout_secs: Optional[float] = IDLE_TIMEOUT_SECS,
         observers: Optional[List[BaseObserver]] = None,
+        rtvi_processor: Optional[RTVIProcessor] = None,
+        rtvi_observer_params: Optional[RTVIObserverParams] = None,
         task_manager: Optional[BaseTaskManager] = None,
     ):
         """Initialize the PipelineTask.
@@ -244,6 +253,7 @@ class PipelineTask(BasePipelineTask):
             check_dangling_tasks: Whether to check for processors' tasks finishing properly.
             clock: Clock implementation for timing operations.
             conversation_id: Optional custom ID for the conversation.
+            enable_rtvi: Whether to automatically add RTVI support to the pipeline.
             enable_tracing: Whether to enable tracing.
             enable_turn_tracking: Whether to enable turn tracking.
             idle_timeout_frames: A tuple with the frames that should trigger an idle
@@ -252,6 +262,8 @@ class PipelineTask(BasePipelineTask):
                 None. If a pipeline is idle the pipeline task will be cancelled
                 automatically.
             observers: List of observers for monitoring pipeline execution.
+            rtvi_observer_params: The RTVI observer parameter to use if RTVI is enabled.
+            rtvi_processor: The RTVI processor to add if RTVI is enabled.
             task_manager: Optional task manager for handling asyncio tasks.
         """
         super().__init__()
@@ -277,15 +289,25 @@ class PipelineTask(BasePipelineTask):
             observers = self._params.observers
         observers = observers or []
         self._turn_tracking_observer: Optional[TurnTrackingObserver] = None
+        self._user_bot_latency_observer: Optional[UserBotLatencyObserver] = None
         self._turn_trace_observer: Optional[TurnTraceObserver] = None
+        self._tracing_context: Optional[TracingContext] = None
         if self._enable_turn_tracking:
             self._turn_tracking_observer = TurnTrackingObserver()
             observers.append(self._turn_tracking_observer)
         if self._enable_tracing and self._turn_tracking_observer:
+            # Create pipeline-scoped tracing context
+            self._tracing_context = TracingContext()
+            # Create latency observer for tracing
+            self._user_bot_latency_observer = UserBotLatencyObserver()
+            observers.append(self._user_bot_latency_observer)
+            # Create turn trace observer with latency tracking
             self._turn_trace_observer = TurnTraceObserver(
                 self._turn_tracking_observer,
+                latency_tracker=self._user_bot_latency_observer,
                 conversation_id=self._conversation_id,
                 additional_span_attributes=self._additional_span_attributes,
+                tracing_context=self._tracing_context,
             )
             observers.append(self._turn_trace_observer)
 
@@ -306,6 +328,39 @@ class PipelineTask(BasePipelineTask):
         self._heartbeat_push_task: Optional[asyncio.Task] = None
         self._heartbeat_monitor_task: Optional[asyncio.Task] = None
 
+        # RTVI support
+        self._rtvi = None
+        prepend_rtvi = False
+        external_rtvi = self._find_processor(pipeline, RTVIProcessor)
+        external_observer_found = any(isinstance(o, RTVIObserver) for o in observers)
+
+        if external_rtvi and not external_observer_found:
+            logger.error(
+                f"{self}: RTVIProcessor found in pipeline but no RTVIObserver in observers. "
+                "Make sure to add both."
+            )
+        elif not external_rtvi and external_observer_found:
+            logger.error(
+                f"{self}: RTVIObserver found in observers but no RTVIProcessor in pipeline. "
+                "Make sure to add both."
+            )
+        elif external_rtvi and external_observer_found:
+            logger.warning(
+                f"{self}: RTVIProcessor and RTVIObserver found, skipping default ones. "
+                "They are both added by default, no need to add them yourself."
+            )
+            self._rtvi = external_rtvi
+        elif enable_rtvi:
+            self._rtvi = rtvi_processor or RTVIProcessor()
+            observers.append(self._rtvi.create_rtvi_observer(params=rtvi_observer_params))
+            prepend_rtvi = True
+
+        if self._rtvi:
+            # Automatically call RTVIProcessor.set_bot_ready()
+            @self.rtvi.event_handler("on_client_ready")
+            async def on_client_ready(rtvi: RTVIProcessor):
+                await rtvi.set_bot_ready()
+
         # This is the idle event. When selected frames are pushed from any
         # processor we consider the pipeline is not idle. We use an observer
         # which will be listening any part of the pipeline.
@@ -334,8 +389,12 @@ class PipelineTask(BasePipelineTask):
         # source allows us to receive and react to upstream frames, and the sink
         # allows us to receive and react to downstream frames.
         source = PipelineSource(self._source_push_frame, name=f"{self}::Source")
-        sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
-        self._pipeline = Pipeline([pipeline], source=source, sink=sink)
+        self._sink = PipelineSink(self._sink_push_frame, name=f"{self}::Sink")
+        # Only prepend the RTVIProcessor if we created it ourselves. When the
+        # user already placed it inside their pipeline we must not insert it
+        # again or it will appear twice in the frame chain.
+        processors = [self._rtvi, pipeline] if prepend_rtvi else [pipeline]
+        self._pipeline = Pipeline(processors, source=source, sink=self._sink)
 
         # The task observer acts as a proxy to the provided observers. This way,
         # we only need to pass a single observer (using the StartFrame) which
@@ -348,8 +407,8 @@ class PipelineTask(BasePipelineTask):
         # in. This is mainly for efficiency reason because each event handler
         # creates a task and most likely you only care about one or two frame
         # types.
-        self._reached_upstream_types: Tuple[Type[Frame], ...] = ()
-        self._reached_downstream_types: Tuple[Type[Frame], ...] = ()
+        self._reached_upstream_types: Set[Type[Frame]] = set()
+        self._reached_downstream_types: Set[Type[Frame]] = set()
         self._register_event_handler("on_frame_reached_upstream")
         self._register_event_handler("on_frame_reached_downstream")
         self._register_event_handler("on_idle_timeout")
@@ -398,6 +457,35 @@ class PipelineTask(BasePipelineTask):
         """
         return self._turn_trace_observer
 
+    @property
+    def rtvi(self) -> RTVIProcessor:
+        """Get the RTVI processor if RTVI is enabled.
+
+        Returns:
+            The RTVI processor added to the pipeline when RTVI is enabled.
+        """
+        if not self._rtvi:
+            raise Exception(f"{self} RTVI is not enabled.")
+        return self._rtvi
+
+    @property
+    def reached_upstream_types(self) -> Tuple[Type[Frame], ...]:
+        """Get the currently configured upstream frame type filters.
+
+        Returns:
+            Tuple of frame types that trigger the on_frame_reached_upstream event.
+        """
+        return tuple(self._reached_upstream_types)
+
+    @property
+    def reached_downstream_types(self) -> Tuple[Type[Frame], ...]:
+        """Get the currently configured downstream frame type filters.
+
+        Returns:
+            Tuple of frame types that trigger the on_frame_reached_downstream event.
+        """
+        return tuple(self._reached_downstream_types)
+
     def event_handler(self, event_name: str):
         """Decorator for registering event handlers.
 
@@ -441,7 +529,7 @@ class PipelineTask(BasePipelineTask):
         Args:
             types: Tuple of frame types to monitor for upstream events.
         """
-        self._reached_upstream_types = types
+        self._reached_upstream_types = set(types)
 
     def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
         """Set which frame types trigger the on_frame_reached_downstream event.
@@ -449,7 +537,23 @@ class PipelineTask(BasePipelineTask):
         Args:
             types: Tuple of frame types to monitor for downstream events.
         """
-        self._reached_downstream_types = types
+        self._reached_downstream_types = set(types)
+
+    def add_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]):
+        """Add frame types to trigger the on_frame_reached_upstream event.
+
+        Args:
+            types: Tuple of frame types to add to upstream monitoring.
+        """
+        self._reached_upstream_types.update(types)
+
+    def add_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
+        """Add frame types to trigger the on_frame_reached_downstream event.
+
+        Args:
+            types: Tuple of frame types to add to downstream monitoring.
+        """
+        self._reached_downstream_types.update(types)
 
     def has_finished(self) -> bool:
         """Check if the pipeline task has finished execution.
@@ -521,26 +625,43 @@ class PipelineTask(BasePipelineTask):
             self._finished = True
             logger.debug(f"Pipeline task {self} has finished")
 
-    async def queue_frame(self, frame: Frame):
-        """Queue a single frame to be pushed down the pipeline.
+    async def queue_frame(
+        self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
+    ):
+        """Queue a single frame to be pushed through the pipeline.
+
+        Downstream frames are pushed from the beginning of the pipeline.
+        Upstream frames are pushed from the end of the pipeline.
 
         Args:
             frame: The frame to be processed.
+            direction: The direction to push the frame. Defaults to downstream.
         """
-        await self._push_queue.put(frame)
+        if direction == FrameDirection.DOWNSTREAM:
+            await self._push_queue.put(frame)
+        else:
+            await self._sink.queue_frame(frame, direction)
 
-    async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
-        """Queues multiple frames to be pushed down the pipeline.
+    async def queue_frames(
+        self,
+        frames: Iterable[Frame] | AsyncIterable[Frame],
+        direction: FrameDirection = FrameDirection.DOWNSTREAM,
+    ):
+        """Queue multiple frames to be pushed through the pipeline.
+
+        Downstream frames are pushed from the beginning of the pipeline.
+        Upstream frames are pushed from the end of the pipeline.
 
         Args:
             frames: An iterable or async iterable of frames to be processed.
+            direction: The direction to push the frames. Defaults to downstream.
         """
         if isinstance(frames, AsyncIterable):
             async for frame in frames:
-                await self.queue_frame(frame)
+                await self.queue_frame(frame, direction)
         elif isinstance(frames, Iterable):
             for frame in frames:
-                await self.queue_frame(frame)
+                await self.queue_frame(frame, direction)
 
     async def _cancel(self, *, reason: Optional[str] = None):
         """Internal cancellation logic for the pipeline task.
@@ -719,6 +840,7 @@ class PipelineTask(BasePipelineTask):
             enable_usage_metrics=self._params.enable_usage_metrics,
             report_only_initial_ttfb=self._params.report_only_initial_ttfb,
             interruption_strategies=self._params.interruption_strategies,
+            tracing_context=self._tracing_context,
         )
         start_frame.metadata = self._create_start_metadata()
         await self._pipeline.queue_frame(start_frame)
@@ -749,27 +871,27 @@ class PipelineTask(BasePipelineTask):
         pipeline to be stopped (e.g. EndTaskFrame) in which case we would send
         an EndFrame down the pipeline.
         """
-        if isinstance(frame, self._reached_upstream_types):
+        if isinstance(frame, tuple(self._reached_upstream_types)):
             await self._call_event_handler("on_frame_reached_upstream", frame)
 
         if isinstance(frame, EndTaskFrame):
             # Tell the task we should end nicely.
-            logger.debug(f"{self}: received end task frame {frame}")
+            logger.debug(f"{self}: received end task frame upstream {frame}")
             await self.queue_frame(EndFrame(reason=frame.reason))
         elif isinstance(frame, CancelTaskFrame):
             # Tell the task we should end right away.
-            logger.debug(f"{self}: received cancel task frame {frame}")
+            logger.debug(f"{self}: received cancel task frame upstream {frame}")
             await self.queue_frame(CancelFrame(reason=frame.reason))
         elif isinstance(frame, StopTaskFrame):
             # Tell the task we should stop nicely.
-            logger.debug(f"{self}: received stop task frame {frame}")
+            logger.debug(f"{self}: received stop task frame upstream {frame}")
             await self.queue_frame(StopFrame())
         elif isinstance(frame, InterruptionTaskFrame):
             # Tell the task we should interrupt the pipeline. Note that we are
             # bypassing the push queue and directly queue into the
             # pipeline. This is in case the push task is blocked waiting for a
             # pipeline-ending frame to finish traversing the pipeline.
-            logger.debug(f"{self}: received interruption task frame {frame}")
+            logger.debug(f"{self}: received interruption task frame upstream {frame}")
             await self._pipeline.queue_frame(InterruptionFrame())
         elif isinstance(frame, ErrorFrame):
             await self._call_event_handler("on_pipeline_error", frame)
@@ -788,11 +910,12 @@ class PipelineTask(BasePipelineTask):
         processors have handled the EndFrame and therefore we can exit the task
         cleanly.
         """
-        if isinstance(frame, self._reached_downstream_types):
+        if isinstance(frame, tuple(self._reached_downstream_types)):
             await self._call_event_handler("on_frame_reached_downstream", frame)
 
         if isinstance(frame, StartFrame):
             await self._call_event_handler("on_pipeline_started", frame)
+            await self._observer.on_pipeline_started()
 
             # Start heartbeat tasks now that StartFrame has been processed
             # by all processors in the pipeline
@@ -811,6 +934,18 @@ class PipelineTask(BasePipelineTask):
             self._pipeline_end_event.set()
         elif isinstance(frame, HeartbeatFrame):
             await self._heartbeat_queue.put(frame)
+        elif isinstance(frame, EndTaskFrame):
+            logger.debug(f"{self}: received end task frame downstream {frame}")
+            await self.queue_frame(EndTaskFrame(reason=frame.reason), FrameDirection.UPSTREAM)
+        elif isinstance(frame, StopTaskFrame):
+            logger.debug(f"{self}: received stop task frame downstream {frame}")
+            await self.queue_frame(StopTaskFrame(), FrameDirection.UPSTREAM)
+        elif isinstance(frame, CancelTaskFrame):
+            logger.debug(f"{self}: received cancel task frame downstream {frame}")
+            await self.queue_frame(CancelTaskFrame(reason=frame.reason), FrameDirection.UPSTREAM)
+        elif isinstance(frame, InterruptionTaskFrame):
+            logger.debug(f"{self}: received interruption task frame downstream {frame}")
+            await self.queue_frame(InterruptionTaskFrame(), FrameDirection.UPSTREAM)
 
     async def _heartbeat_push_handler(self):
         """Push heartbeat frames at regular intervals."""
@@ -949,7 +1084,7 @@ class PipelineTask(BasePipelineTask):
         start_metadata = {}
 
         # NOTE(aleix): Remove when OpenAILLMContext/LLMUserContextAggregator is removed.
-        if self._find_deprecated_openaillmcontext(self._pipeline):
+        if self._find_processor(self._pipeline, LLMUserContextAggregator):
             start_metadata["deprecated_openaillmcontext"] = True
 
         # Update with user provided metadata.
@@ -957,12 +1092,13 @@ class PipelineTask(BasePipelineTask):
 
         return start_metadata
 
-    def _find_deprecated_openaillmcontext(self, processor: FrameProcessor) -> bool:
-        """Check whether there is a deprecated LLMUserContextAggregator in the pipeline."""
-        if isinstance(processor, LLMUserContextAggregator):
-            return True
+    def _find_processor(self, processor: FrameProcessor, processor_type: Type[T]) -> Optional[T]:
+        """Recursively find a processor of the given type in the pipeline."""
+        if isinstance(processor, processor_type):
+            return processor
 
         for p in processor.processors:
-            if self._find_deprecated_openaillmcontext(p):
-                return True
-        return False
+            found = self._find_processor(p, processor_type)
+            if found:
+                return found
+        return None

src/pipecat/pipeline/task_observer.py

@@ -39,6 +39,12 @@ class Proxy:
     observer: BaseObserver
 
 
+class _PipelineStartedSignal:
+    """Internal sentinel queued to observers when the pipeline has started."""
+
+    pass
+
+
 class TaskObserver(BaseObserver):
     """Proxy observer that manages multiple observers without blocking the pipeline.
 
@@ -129,6 +135,10 @@ class TaskObserver(BaseObserver):
         for proxy in self._proxies:
             await proxy.cleanup()
 
+    async def on_pipeline_started(self):
+        """Forward pipeline started signal to all managed observers."""
+        await self._send_to_proxy(_PipelineStartedSignal())
+
     async def on_process_frame(self, data: FrameProcessed):
         """Queue frame data for all managed observers.
 
@@ -186,7 +196,9 @@ class TaskObserver(BaseObserver):
         while True:
             data = await queue.get()
 
-            if isinstance(data, FramePushed):
+            if isinstance(data, _PipelineStartedSignal):
+                await observer.on_pipeline_started()
+            elif isinstance(data, FramePushed):
                 if on_push_frame_deprecated:
                     await observer.on_push_frame(
                         data.source, data.destination, data.frame, data.direction, data.timestamp

src/pipecat/pipeline/to_be_updated/merge_pipeline.py

@@ -1,53 +0,0 @@
-#
-# Copyright (c) 2024-2026, Daily
-#
-# SPDX-License-Identifier: BSD 2-Clause License
-#
-
-"""Sequential pipeline merging for Pipecat.
-
-This module provides a pipeline implementation that sequentially merges
-the output from multiple pipelines, processing them one after another
-in a specified order.
-"""
-
-from typing import List
-
-from pipecat.frames.frames import EndFrame, EndPipeFrame
-from pipecat.pipeline.pipeline import Pipeline
-
-
-class SequentialMergePipeline(Pipeline):
-    """Pipeline that sequentially merges output from multiple pipelines.
-
-    This pipeline merges the sink queues from a list of pipelines by processing
-    frames from each pipeline's sink sequentially in the order specified. Each
-    pipeline runs to completion before the next one begins processing.
-    """
-
-    def __init__(self, pipelines: List[Pipeline]):
-        """Initialize the sequential merge pipeline.
-
-        Args:
-            pipelines: List of pipelines to merge sequentially. Pipelines will
-                      be processed in the order they appear in this list.
-        """
-        super().__init__([])
-        self.pipelines = pipelines
-
-    async def run_pipeline(self):
-        """Run all pipelines sequentially and merge their output.
-
-        Processes each pipeline in order, consuming all frames from each
-        pipeline's sink until an EndFrame or EndPipeFrame is encountered,
-        then moves to the next pipeline. After all pipelines complete,
-        sends a final EndFrame to signal completion.
-        """
-        for idx, pipeline in enumerate(self.pipelines):
-            while True:
-                frame = await pipeline.sink.get()
-                if isinstance(frame, EndFrame) or isinstance(frame, EndPipeFrame):
-                    break
-                await self.sink.put(frame)
-
-        await self.sink.put(EndFrame())

src/pipecat/processors/aggregators/dtmf_aggregator.py

@@ -104,7 +104,7 @@ class DTMFAggregator(FrameProcessor):
 
         # For first digit, schedule interruption.
         if is_first_digit:
-            await self.push_interruption_task_frame_and_wait()
+            await self.broadcast_interruption()
 
         # Check for immediate flush conditions
         if frame.button == self._termination_digit:

src/pipecat/processors/aggregators/llm_context.py

@@ -206,7 +206,7 @@ class LLMContext:
         """
         content = [{"type": "text", "text": text}]
 
-        async def encode_audio():
+        def encode_audio():
             sample_rate = audio_frames[0].sample_rate
             num_channels = audio_frames[0].num_channels
 
@@ -255,7 +255,7 @@ class LLMContext:
         this method, which is part of the public API of OpenAILLMContext but
         doesn't need to be for LLMContext.
 
-        .. deprecated::
+        .. deprecated:: 0.0.92
             Use `get_messages()` instead.
 
         Returns:

src/pipecat/processors/aggregators/llm_context_summarizer.py

@@ -0,0 +1,480 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""This module defines a summarizer for managing LLM context summarization."""
+
+import asyncio
+import uuid
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Optional
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+    Frame,
+    InterruptionFrame,
+    LLMContextSummaryRequestFrame,
+    LLMContextSummaryResultFrame,
+    LLMFullResponseStartFrame,
+    LLMSummarizeContextFrame,
+)
+from pipecat.processors.aggregators.llm_context import LLMContext, LLMSpecificMessage
+from pipecat.utils.asyncio.task_manager import BaseTaskManager
+from pipecat.utils.base_object import BaseObject
+from pipecat.utils.context.llm_context_summarization import (
+    DEFAULT_SUMMARIZATION_TIMEOUT,
+    LLMAutoContextSummarizationConfig,
+    LLMContextSummarizationUtil,
+    LLMContextSummaryConfig,
+)
+
+if TYPE_CHECKING:
+    from pipecat.services.llm_service import LLMService
+
+
+@dataclass
+class SummaryAppliedEvent:
+    """Event data emitted when context summarization completes successfully.
+
+    Parameters:
+        original_message_count: Number of messages before summarization.
+        new_message_count: Number of messages after summarization.
+        summarized_message_count: Number of messages that were compressed
+            into the summary.
+        preserved_message_count: Number of recent messages preserved
+            uncompressed.
+    """
+
+    original_message_count: int
+    new_message_count: int
+    summarized_message_count: int
+    preserved_message_count: int
+
+
+class LLMContextSummarizer(BaseObject):
+    """Summarizer for managing LLM context summarization.
+
+    This class manages context summarization, either automatically when token or
+    message limits are reached, or on-demand when an ``LLMSummarizeContextFrame``
+    is received. It monitors the LLM context size, triggers summarization requests,
+    and applies the results to compress conversation history.
+
+    When ``auto_trigger=True`` (the default), summarization is triggered
+    automatically based on the configured thresholds in
+    ``LLMAutoContextSummarizationConfig``. When ``auto_trigger=False``,
+    threshold checks are skipped and summarization only happens when an
+    ``LLMSummarizeContextFrame`` is explicitly pushed into the pipeline.
+
+    Both modes can coexist: set ``auto_trigger=True`` and also push
+    ``LLMSummarizeContextFrame`` at any time to force an immediate summarization
+    (subject to the ``_summarization_in_progress`` guard).
+
+    Event handlers available:
+
+    - on_request_summarization: Emitted when summarization should be triggered.
+        The aggregator should broadcast this frame to the LLM service.
+
+    - on_summary_applied: Emitted after a summary has been successfully applied
+        to the context. Receives a SummaryAppliedEvent with metrics about the
+        compression.
+
+    Example::
+
+        @summarizer.event_handler("on_request_summarization")
+        async def on_request_summarization(summarizer, frame: LLMContextSummaryRequestFrame):
+            await aggregator.broadcast_frame(
+                LLMContextSummaryRequestFrame,
+                request_id=frame.request_id,
+                context=frame.context,
+                ...
+            )
+
+        @summarizer.event_handler("on_summary_applied")
+        async def on_summary_applied(summarizer, event: SummaryAppliedEvent):
+            logger.info(f"Compressed {event.original_message_count} -> {event.new_message_count} messages")
+    """
+
+    def __init__(
+        self,
+        *,
+        context: LLMContext,
+        config: Optional[LLMAutoContextSummarizationConfig] = None,
+        auto_trigger: bool = True,
+    ):
+        """Initialize the context summarizer.
+
+        Args:
+            context: The LLM context to monitor and summarize.
+            config: Auto-summarization configuration controlling both trigger
+                thresholds and default summary generation parameters. If None,
+                uses default ``LLMAutoContextSummarizationConfig`` values.
+            auto_trigger: Whether to automatically trigger summarization when
+                thresholds are reached. When False, summarization only happens
+                when an ``LLMSummarizeContextFrame`` is pushed into the pipeline.
+                Defaults to True.
+        """
+        super().__init__()
+
+        self._context = context
+        self._auto_config = config or LLMAutoContextSummarizationConfig()
+        self._auto_trigger = auto_trigger
+
+        self._task_manager: Optional[BaseTaskManager] = None
+
+        self._summarization_in_progress = False
+        self._pending_summary_request_id: Optional[str] = None
+
+        self._register_event_handler("on_request_summarization", sync=True)
+        self._register_event_handler("on_summary_applied")
+
+    @property
+    def task_manager(self) -> BaseTaskManager:
+        """Returns the configured task manager."""
+        if not self._task_manager:
+            raise RuntimeError(f"{self} context summarizer was not properly setup")
+        return self._task_manager
+
+    async def setup(self, task_manager: BaseTaskManager):
+        """Initialize the summarizer with the given task manager.
+
+        Args:
+            task_manager: The task manager to be associated with this instance.
+        """
+        self._task_manager = task_manager
+
+    async def cleanup(self):
+        """Cleanup the summarizer."""
+        await super().cleanup()
+        await self._clear_summarization_state()
+
+    async def process_frame(self, frame: Frame):
+        """Process an incoming frame to detect when summarization is needed.
+
+        Args:
+            frame: The frame to be processed.
+        """
+        if isinstance(frame, LLMFullResponseStartFrame):
+            await self._handle_llm_response_start(frame)
+        elif isinstance(frame, LLMSummarizeContextFrame):
+            await self._handle_manual_summarization_request(frame)
+        elif isinstance(frame, LLMContextSummaryResultFrame):
+            await self._handle_summary_result(frame)
+        elif isinstance(frame, InterruptionFrame):
+            await self._handle_interruption()
+
+    async def _handle_llm_response_start(self, frame: LLMFullResponseStartFrame):
+        """Handle LLM response start to check if summarization is needed.
+
+        Args:
+            frame: The LLM response start frame.
+        """
+        if self._should_summarize():
+            await self._request_summarization()
+
+    async def _handle_manual_summarization_request(self, frame: LLMSummarizeContextFrame):
+        """Handle an explicit on-demand summarization request.
+
+        Reuses the same ``_request_summarization()`` code path as auto mode,
+        so bookkeeping (``_summarization_in_progress``,
+        ``_pending_summary_request_id``) is always updated correctly.
+
+        Args:
+            frame: The manual summarization request frame, optionally carrying
+                a per-request :class:`~pipecat.utils.context.llm_context_summarization.LLMContextSummaryConfig`.
+        """
+        if self._summarization_in_progress:
+            logger.debug(f"{self}: Summarization already in progress, ignoring manual request")
+            return
+        await self._request_summarization(config_override=frame.config)
+
+    async def _handle_interruption(self):
+        """Handle interruption by canceling summarization in progress."""
+        # Reset summarization state to allow new requests. This is necessary because
+        # the request frame (LLMContextSummaryRequestFrame) may have been cancelled
+        # during interruption. We preserve _pending_summary_request_id to handle the
+        # response frame (LLMContextSummaryResultFrame), which is uninterruptible and
+        # will still be delivered.
+        self._summarization_in_progress = False
+
+    async def _clear_summarization_state(self):
+        """Cancel pending summarization."""
+        if self._summarization_in_progress:
+            logger.debug(f"{self}: Clearing pending summarization")
+            self._summarization_in_progress = False
+            self._pending_summary_request_id = None
+
+    def _should_summarize(self) -> bool:
+        """Determine if context summarization should be triggered.
+
+        Evaluates whether the current context has reached either the token
+        threshold or message count threshold that warrants compression.
+        Either threshold can be ``None`` to disable that check; at least one
+        must be set (enforced at config construction time).
+
+        Returns:
+            True if all conditions are met:
+            - ``auto_trigger`` is enabled
+            - No summarization currently in progress
+            - AND either:
+              - Token count exceeds ``max_context_tokens`` (when set)
+              - OR message count exceeds ``max_unsummarized_messages`` since last summary (when set)
+        """
+        logger.trace(f"{self}: Checking if context summarization is needed")
+
+        if not self._auto_trigger:
+            return False
+
+        if self._summarization_in_progress:
+            logger.debug(f"{self}: Summarization already in progress")
+            return False
+
+        # Estimate tokens in context
+        total_tokens = LLMContextSummarizationUtil.estimate_context_tokens(self._context)
+        num_messages = len(self._context.messages)
+
+        # Check if we've reached the token limit
+        token_limit = self._auto_config.max_context_tokens
+        token_limit_exceeded = token_limit is not None and total_tokens >= token_limit
+
+        # Check if we've exceeded max unsummarized messages
+        messages_since_summary = len(self._context.messages) - 1
+        message_threshold = self._auto_config.max_unsummarized_messages
+        message_threshold_exceeded = (
+            message_threshold is not None and messages_since_summary >= message_threshold
+        )
+
+        logger.trace(
+            f"{self}: Context has {num_messages} messages, "
+            f"~{total_tokens} tokens (limit: {token_limit if token_limit is not None else 'disabled'}), "
+            f"{messages_since_summary} messages since last summary "
+            f"(message threshold: {message_threshold if message_threshold is not None else 'disabled'})"
+        )
+
+        # Trigger if either limit is exceeded
+        if not token_limit_exceeded and not message_threshold_exceeded:
+            logger.trace(
+                f"{self}: Neither token limit nor message threshold exceeded, skipping summarization"
+            )
+            return False
+
+        reason = []
+        if token_limit_exceeded:
+            reason.append(f"~{total_tokens} tokens (>={token_limit} limit)")
+        if message_threshold_exceeded:
+            reason.append(f"{messages_since_summary} messages (>={message_threshold} threshold)")
+
+        logger.debug(f"{self}: ✓ Summarization needed - {', '.join(reason)}")
+        return True
+
+    async def _request_summarization(
+        self, config_override: Optional[LLMContextSummaryConfig] = None
+    ):
+        """Request context summarization from LLM service.
+
+        Creates a summarization request frame and either handles it directly
+        using a dedicated LLM (if configured) or emits it via event handler
+        for the pipeline's primary LLM.
+        Tracks the request ID to match async responses and prevent race conditions.
+
+        Args:
+            config_override: Optional per-request summary configuration. If provided,
+                overrides the default summary generation settings from
+                ``self._auto_config.summary_config``.
+        """
+        # Generate unique request ID
+        request_id = str(uuid.uuid4())
+        summary_config = config_override or self._auto_config.summary_config
+
+        # Mark summarization in progress
+        self._summarization_in_progress = True
+        self._pending_summary_request_id = request_id
+
+        logger.debug(f"{self}: Sending summarization request (request_id={request_id})")
+
+        # Create the request frame
+        request_frame = LLMContextSummaryRequestFrame(
+            request_id=request_id,
+            context=self._context,
+            min_messages_to_keep=summary_config.min_messages_after_summary,
+            target_context_tokens=summary_config.target_context_tokens,
+            summarization_prompt=summary_config.summary_prompt,
+            summarization_timeout=summary_config.summarization_timeout,
+        )
+
+        if summary_config.llm:
+            # Use dedicated LLM directly — no need to involve the pipeline
+            self.task_manager.create_task(
+                self._generate_summary_with_dedicated_llm(summary_config.llm, request_frame),
+                f"{self}-dedicated-llm-summary",
+            )
+        else:
+            # Emit event for aggregator to broadcast to the pipeline LLM
+            await self._call_event_handler("on_request_summarization", request_frame)
+
+    async def _generate_summary_with_dedicated_llm(
+        self, llm: "LLMService", frame: LLMContextSummaryRequestFrame
+    ):
+        """Generate summary using a dedicated LLM service.
+
+        Calls the dedicated LLM's _generate_summary directly and feeds the
+        result back through _handle_summary_result, bypassing the pipeline.
+
+        Args:
+            llm: The dedicated LLM service to use for summarization.
+            frame: The summarization request frame.
+        """
+        timeout = frame.summarization_timeout or DEFAULT_SUMMARIZATION_TIMEOUT
+
+        try:
+            summary, last_index = await asyncio.wait_for(
+                llm._generate_summary(frame),
+                timeout=timeout,
+            )
+            result_frame = LLMContextSummaryResultFrame(
+                request_id=frame.request_id,
+                summary=summary,
+                last_summarized_index=last_index,
+            )
+        except asyncio.TimeoutError:
+            error = f"Context summarization timed out after {timeout}s"
+            logger.error(f"{self}: {error}")
+            result_frame = LLMContextSummaryResultFrame(
+                request_id=frame.request_id,
+                summary="",
+                last_summarized_index=-1,
+                error=error,
+            )
+        except Exception as e:
+            error = f"Error generating context summary: {e}"
+            logger.error(f"{self}: {error}")
+            result_frame = LLMContextSummaryResultFrame(
+                request_id=frame.request_id,
+                summary="",
+                last_summarized_index=-1,
+                error=error,
+            )
+
+        await self._handle_summary_result(result_frame)
+
+    async def _handle_summary_result(self, frame: LLMContextSummaryResultFrame):
+        """Handle context summarization result from LLM service.
+
+        Processes the summary result by validating the request ID, checking for
+        errors, validating context state, and applying the summary.
+
+        Args:
+            frame: The summary result frame containing the generated summary.
+        """
+        logger.debug(f"{self}: Received summary result (request_id={frame.request_id})")
+
+        # Check if this is the result we're waiting for. Both auto and manual
+        # summarization set _pending_summary_request_id via _request_summarization(),
+        # so this check always applies.
+        if frame.request_id != self._pending_summary_request_id:
+            logger.debug(f"{self}: Ignoring stale summary result (request_id={frame.request_id})")
+            return
+
+        # Clear pending state
+        await self._clear_summarization_state()
+
+        # Check for errors
+        if frame.error:
+            logger.error(f"{self}: Context summarization failed: {frame.error}")
+            return
+
+        # Validate context state
+        if not self._validate_summary_context(frame.last_summarized_index):
+            logger.warning(f"{self}: Context state changed, skipping summary application")
+            return
+
+        # Apply summary
+        await self._apply_summary(frame.summary, frame.last_summarized_index)
+
+    def _validate_summary_context(self, last_summarized_index: int) -> bool:
+        """Validate that context state is still valid for applying summary.
+
+        Args:
+            last_summarized_index: The index of the last summarized message.
+
+        Returns:
+            True if the context state is still consistent with the summary.
+        """
+        if last_summarized_index < 0:
+            return False
+
+        # Check if we still have enough messages
+        if last_summarized_index >= len(self._context.messages):
+            return False
+
+        min_keep = self._auto_config.summary_config.min_messages_after_summary
+        remaining = len(self._context.messages) - 1 - last_summarized_index
+        if remaining < min_keep:
+            return False
+
+        return True
+
+    async def _apply_summary(self, summary: str, last_summarized_index: int):
+        """Apply summary to compress the conversation context.
+
+        Reconstructs the context with:
+        [first_system_message] + [summary_message] + [recent_messages]
+
+        Args:
+            summary: The generated summary text.
+            last_summarized_index: Index of the last message that was summarized.
+        """
+        config = self._auto_config.summary_config
+        messages = self._context.messages
+
+        # Find the first system message to preserve. LLMSpecificMessage instances are excluded
+        # because they are not dict-like and never represent a system message; they hold
+        # service-specific metadata (e.g. thinking blocks) that is always paired with a
+        # standard message.
+        first_system_msg = next(
+            (
+                msg
+                for msg in messages
+                if not isinstance(msg, LLMSpecificMessage) and msg.get("role") == "system"
+            ),
+            None,
+        )
+
+        # Get recent messages to keep
+        recent_messages = messages[last_summarized_index + 1 :]
+
+        # Create summary message as a user message (the summary is context
+        # provided *to* the assistant, not something the assistant said)
+        summary_content = config.summary_message_template.format(summary=summary)
+        summary_message = {"role": "user", "content": summary_content}
+
+        # Reconstruct context
+        new_messages = []
+        if first_system_msg:
+            new_messages.append(first_system_msg)
+        new_messages.append(summary_message)
+        new_messages.extend(recent_messages)
+
+        # Update context
+        original_message_count = len(messages)
+        num_system_preserved = 1 if first_system_msg else 0
+        self._context.set_messages(new_messages)
+
+        # Messages actually summarized = index range minus the preserved system message
+        summarized_count = last_summarized_index + 1 - num_system_preserved
+
+        logger.info(
+            f"{self}: Applied context summary, compressed {summarized_count} messages "
+            f"into summary. Context now has {len(new_messages)} messages (was {original_message_count})"
+        )
+
+        # Emit event for observability
+        event = SummaryAppliedEvent(
+            original_message_count=original_message_count,
+            new_message_count=len(new_messages),
+            summarized_message_count=summarized_count,
+            preserved_message_count=len(recent_messages) + num_system_preserved,
+        )
+        await self._call_event_handler("on_summary_applied", event)

src/pipecat/processors/aggregators/llm_response.py

@@ -581,7 +581,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
                     logger.debug(
                         "Interruption conditions met - pushing interruption and aggregation"
                     )
-                    await self.push_interruption_task_frame_and_wait()
+                    await self.broadcast_interruption()
                     await self._process_aggregation()
                 else:
                     logger.debug("Interruption conditions not met - not pushing aggregation")
@@ -1024,10 +1024,8 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
         logger.debug(
             f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]"
         )
-        if frame.tool_call_id not in self._function_calls_in_progress:
-            return
-
-        if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption:
+        function_call = self._function_calls_in_progress.get(frame.tool_call_id)
+        if function_call and function_call.cancel_on_interruption:
             await self.handle_function_call_cancel(frame)
             del self._function_calls_in_progress[frame.tool_call_id]
 
@@ -1044,6 +1042,11 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
 
         del self._function_calls_in_progress[frame.request.tool_call_id]
 
+        # Call the result_callback if provided. This signals that the image
+        # has been retrieved and the function call can now complete.
+        if frame.request and frame.request.result_callback:
+            await frame.request.result_callback(None)
+
         await self.handle_user_image_frame(frame)
         await self.push_aggregation()
         await self.push_context_frame(FrameDirection.UPSTREAM)
@@ -1056,7 +1059,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
         await self.push_aggregation()
 
     async def _handle_text(self, frame: TextFrame):
-        if not self._started or not frame.append_to_context:
+        if not frame.append_to_context:
             return
 
         if self._params.expect_stripped_words:

src/pipecat/processors/aggregators/llm_response_universal.py

@@ -21,6 +21,8 @@ from typing import Any, Dict, List, Literal, Optional, Set, Type
 from loguru import logger
 
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.audio.vad.vad_analyzer import VADAnalyzer
+from pipecat.audio.vad.vad_controller import VADController
 from pipecat.frames.frames import (
     AssistantImageRawFrame,
     CancelFrame,
@@ -33,8 +35,10 @@ from pipecat.frames.frames import (
     InputAudioRawFrame,
     InterimTranscriptionFrame,
     InterruptionFrame,
+    LLMAssistantPushAggregationFrame,
     LLMContextAssistantTimestampFrame,
     LLMContextFrame,
+    LLMContextSummaryRequestFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesAppendFrame,
@@ -45,11 +49,16 @@ from pipecat.frames.frames import (
     LLMThoughtEndFrame,
     LLMThoughtStartFrame,
     LLMThoughtTextFrame,
+    LLMUpdateSettingsFrame,
     SpeechControlParamsFrame,
     StartFrame,
     TextFrame,
     TranscriptionFrame,
+    TranslationFrame,
     UserImageRawFrame,
+    UserMuteStartedFrame,
+    UserMuteStoppedFrame,
+    UserSpeakingFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
     VADUserStartedSpeakingFrame,
@@ -61,12 +70,23 @@ from pipecat.processors.aggregators.llm_context import (
     LLMSpecificMessage,
     NotGiven,
 )
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
-from pipecat.turns.mute import BaseUserMuteStrategy
+from pipecat.processors.aggregators.llm_context_summarizer import (
+    LLMContextSummarizer,
+    SummaryAppliedEvent,
+)
+from pipecat.processors.frame_processor import FrameCallback, FrameDirection, FrameProcessor
+from pipecat.services.settings import LLMSettings
+from pipecat.turns.user_idle_controller import UserIdleController
+from pipecat.turns.user_mute import BaseUserMuteStrategy
 from pipecat.turns.user_start import BaseUserTurnStartStrategy, UserTurnStartedParams
 from pipecat.turns.user_stop import BaseUserTurnStopStrategy, UserTurnStoppedParams
+from pipecat.turns.user_turn_completion_mixin import UserTurnCompletionConfig
 from pipecat.turns.user_turn_controller import UserTurnController
 from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies, UserTurnStrategies
+from pipecat.utils.context.llm_context_summarization import (
+    LLMAutoContextSummarizationConfig,
+    LLMContextSummarizationConfig,
+)
 from pipecat.utils.string import TextPartForConcatenation, concatenate_aggregated_text
 from pipecat.utils.time import time_now_iso8601
 
@@ -80,11 +100,27 @@ class LLMUserAggregatorParams:
         user_mute_strategies: List of user mute strategies.
         user_turn_stop_timeout: Time in seconds to wait before considering the
             user's turn finished.
+        user_idle_timeout: Timeout in seconds for detecting user idle state.
+            The aggregator will emit an `on_user_turn_idle` event when the user
+            has been idle (not speaking) for this duration. Set to 0 to disable
+            idle detection.
+        vad_analyzer: Voice Activity Detection analyzer instance.
+        filter_incomplete_user_turns: Whether to filter out incomplete user turns.
+            When enabled, the LLM outputs a turn completion marker at the start of
+            each response: ✓ (complete), ○ (incomplete short), or ◐ (incomplete long).
+            Incomplete responses are suppressed and timeouts trigger re-prompting.
+        user_turn_completion_config: Configuration for turn completion behavior including
+            custom instructions, timeouts, and prompts. Only used when
+            filter_incomplete_user_turns is True.
     """
 
     user_turn_strategies: Optional[UserTurnStrategies] = None
     user_mute_strategies: List[BaseUserMuteStrategy] = field(default_factory=list)
     user_turn_stop_timeout: float = 5.0
+    user_idle_timeout: float = 0
+    vad_analyzer: Optional[VADAnalyzer] = None
+    filter_incomplete_user_turns: bool = False
+    user_turn_completion_config: Optional[UserTurnCompletionConfig] = None
 
 
 @dataclass
@@ -96,9 +132,53 @@ class LLMAssistantAggregatorParams:
             in text frames by adding spaces between tokens. This parameter is
             ignored when used with the newer LLMAssistantAggregator, which
             handles word spacing automatically.
+        enable_auto_context_summarization: Enable automatic context summarization when token
+            or message-count limits are reached (disabled by default). When enabled,
+            older conversation messages are automatically compressed into summaries to
+            manage context size.
+        auto_context_summarization_config: Configuration for automatic context
+            summarization. Controls trigger thresholds, message preservation, and
+            summarization prompts. If None, uses default
+            ``LLMAutoContextSummarizationConfig`` values.
     """
 
     expect_stripped_words: bool = True
+    enable_auto_context_summarization: bool = False
+    auto_context_summarization_config: Optional[LLMAutoContextSummarizationConfig] = None
+
+    # ---------------------------------------------------------------------------
+    # Deprecated field names — kept for backward compatibility.
+    # Use enable_auto_context_summarization and auto_context_summarization_config instead.
+    # ---------------------------------------------------------------------------
+    enable_context_summarization: Optional[bool] = None
+    context_summarization_config: Optional[LLMContextSummarizationConfig] = None
+
+    def __post_init__(self):
+        if self.enable_context_summarization is not None:
+            warnings.warn(
+                "LLMAssistantAggregatorParams.enable_context_summarization is deprecated. "
+                "Use enable_auto_context_summarization instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            self.enable_auto_context_summarization = self.enable_context_summarization
+            self.enable_context_summarization = None
+
+        if self.context_summarization_config is not None:
+            warnings.warn(
+                "LLMAssistantAggregatorParams.context_summarization_config is deprecated. "
+                "Use auto_context_summarization_config (LLMAutoContextSummarizationConfig) instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
+            if isinstance(self.context_summarization_config, LLMContextSummarizationConfig):
+                self.auto_context_summarization_config = (
+                    self.context_summarization_config.to_auto_config()
+                )
+            else:
+                # Accept LLMAutoContextSummarizationConfig passed to the deprecated field
+                self.auto_context_summarization_config = self.context_summarization_config  # type: ignore[assignment]
+            self.context_summarization_config = None
 
 
 @dataclass
@@ -291,11 +371,14 @@ class LLMUserAggregator(LLMContextAggregator):
     - on_user_turn_started: Called when the user turn starts
     - on_user_turn_stopped: Called when the user turn ends
     - on_user_turn_stop_timeout: Called when no user turn stop strategy triggers
+    - on_user_turn_idle: Called when the user has been idle for the configured timeout
+    - on_user_mute_started: Called when the user becomes muted
+    - on_user_mute_stopped: Called when the user becomes unmuted
 
     Example::
 
         @aggregator.event_handler("on_user_turn_started")
-        async def on_user_turn_started(aggregator, strategy: BaseUserTurnStartStrategy]):
+        async def on_user_turn_started(aggregator, strategy: BaseUserTurnStartStrategy):
             ...
 
         @aggregator.event_handler("on_user_turn_stopped")
@@ -306,6 +389,18 @@ class LLMUserAggregator(LLMContextAggregator):
         async def on_user_turn_stop_timeout(aggregator):
             ...
 
+        @aggregator.event_handler("on_user_turn_idle")
+        async def on_user_turn_idle(aggregator):
+            ...
+
+        @aggregator.event_handler("on_user_mute_started")
+        async def on_user_mute_started(aggregator):
+            ...
+
+        @aggregator.event_handler("on_user_mute_stopped")
+        async def on_user_mute_stopped(aggregator):
+            ...
+
     """
 
     def __init__(
@@ -328,6 +423,9 @@ class LLMUserAggregator(LLMContextAggregator):
         self._register_event_handler("on_user_turn_started")
         self._register_event_handler("on_user_turn_stopped")
         self._register_event_handler("on_user_turn_stop_timeout")
+        self._register_event_handler("on_user_turn_idle")
+        self._register_event_handler("on_user_mute_started")
+        self._register_event_handler("on_user_mute_stopped")
 
         user_turn_strategies = self._params.user_turn_strategies or UserTurnStrategies()
 
@@ -349,6 +447,31 @@ class LLMUserAggregator(LLMContextAggregator):
         self._user_turn_controller.add_event_handler(
             "on_user_turn_stop_timeout", self._on_user_turn_stop_timeout
         )
+        self._user_turn_controller.add_event_handler(
+            "on_reset_aggregation", self._on_reset_aggregation
+        )
+
+        self._user_idle_controller = UserIdleController(
+            user_idle_timeout=self._params.user_idle_timeout
+        )
+        self._user_idle_controller.add_event_handler("on_user_turn_idle", self._on_user_turn_idle)
+
+        # VAD controller
+        self._vad_controller: Optional[VADController] = None
+        if self._params.vad_analyzer:
+            self._vad_controller = VADController(self._params.vad_analyzer)
+            self._vad_controller.add_event_handler("on_speech_started", self._on_vad_speech_started)
+            self._vad_controller.add_event_handler("on_speech_stopped", self._on_vad_speech_stopped)
+            self._vad_controller.add_event_handler(
+                "on_speech_activity", self._on_vad_speech_activity
+            )
+            self._vad_controller.add_event_handler("on_push_frame", self._on_push_frame)
+            self._vad_controller.add_event_handler("on_broadcast_frame", self._on_broadcast_frame)
+
+        # NOTE(aleix): Probably just needed temporarily. This was added to
+        # prevent processing self-queued frames (SpeechControlParamsFrame)
+        # pushed by strategies.
+        self._self_queued_frames = set()
 
     async def cleanup(self):
         """Clean up processor resources."""
@@ -367,6 +490,9 @@ class LLMUserAggregator(LLMContextAggregator):
         if await self._maybe_mute_frame(frame):
             return
 
+        if self._vad_controller:
+            await self._vad_controller.process_frame(frame)
+
         if isinstance(frame, StartFrame):
             # Push StartFrame before start(), because we want StartFrame to be
             # processed by every processor before any other frame is processed.
@@ -382,6 +508,10 @@ class LLMUserAggregator(LLMContextAggregator):
             await self.push_frame(frame, direction)
         elif isinstance(frame, TranscriptionFrame):
             await self._handle_transcription(frame)
+        elif isinstance(frame, (InterimTranscriptionFrame, TranslationFrame)):
+            # Interim transcriptions and translations are consumed here
+            # and not pushed downstream, same as final TranscriptionFrame.
+            pass
         elif isinstance(frame, LLMRunFrame):
             await self._handle_llm_run(frame)
         elif isinstance(frame, LLMMessagesAppendFrame):
@@ -405,6 +535,8 @@ class LLMUserAggregator(LLMContextAggregator):
 
         await self._user_turn_controller.process_frame(frame)
 
+        await self._user_idle_controller.process_frame(frame)
+
     async def push_aggregation(self) -> str:
         """Push the current aggregation."""
         if len(self._aggregation) == 0:
@@ -420,22 +552,49 @@ class LLMUserAggregator(LLMContextAggregator):
     async def _start(self, frame: StartFrame):
         await self._user_turn_controller.setup(self.task_manager)
 
+        await self._user_idle_controller.setup(self.task_manager)
+
         for s in self._params.user_mute_strategies:
             await s.setup(self.task_manager)
 
+        # Enable incomplete turn filtering on the LLM if configured
+        if self._params.filter_incomplete_user_turns:
+            # Get config or use defaults
+            config = self._params.user_turn_completion_config or UserTurnCompletionConfig()
+
+            # Enable the feature on the LLM with config
+            await self.push_frame(
+                LLMUpdateSettingsFrame(
+                    delta=LLMSettings(
+                        filter_incomplete_user_turns=True,
+                        user_turn_completion_config=config,
+                    )
+                )
+            )
+
     async def _stop(self, frame: EndFrame):
+        await self._maybe_emit_user_turn_stopped(on_session_end=True)
         await self._cleanup()
 
     async def _cancel(self, frame: CancelFrame):
+        await self._maybe_emit_user_turn_stopped(on_session_end=True)
         await self._cleanup()
 
     async def _cleanup(self):
         await self._user_turn_controller.cleanup()
+        await self._user_idle_controller.cleanup()
 
         for s in self._params.user_mute_strategies:
             await s.cleanup()
 
     async def _maybe_mute_frame(self, frame: Frame):
+        # Lifecycle frames should never be muted and should not trigger mute
+        # state changes. Evaluating mute strategies on StartFrame would
+        # broadcast UserMuteStartedFrame before StartFrame reaches downstream
+        # processors.
+        if isinstance(frame, (StartFrame, EndFrame, CancelFrame)):
+            return False
+
         should_mute_frame = self._user_is_muted and isinstance(
             frame,
             (
@@ -461,6 +620,14 @@ class LLMUserAggregator(LLMContextAggregator):
             logger.debug(f"{self}: user is now {'muted' if should_mute_next_time else 'unmuted'}")
             self._user_is_muted = should_mute_next_time
 
+            # Emit mute state change events
+            if self._user_is_muted:
+                await self._call_event_handler("on_user_mute_started")
+                await self.broadcast_frame(UserMuteStartedFrame)
+            else:
+                await self._call_event_handler("on_user_mute_stopped")
+                await self.broadcast_frame(UserMuteStoppedFrame)
+
         return should_mute_frame
 
     async def _handle_llm_run(self, frame: LLMRunFrame):
@@ -477,28 +644,13 @@ class LLMUserAggregator(LLMContextAggregator):
             await self.push_context_frame()
 
     async def _handle_speech_control_params(self, frame: SpeechControlParamsFrame):
+        if frame.id in self._self_queued_frames:
+            return
+
         if not frame.turn_params:
             return
 
-        logger.warning(
-            f"{self}: `turn_analyzer` in base input transport is deprecated. "
-            "Use `LLMUserAggregator`'s new `user_turn_strategies` parameter with "
-            "`TurnAnalyzerUserTurnStopStrategy` instead:\n"
-            "\n"
-            "    context_aggregator = LLMContextAggregatorPair(\n"
-            "        context,\n"
-            "        user_params=LLMUserAggregatorParams(\n"
-            "            ...,\n"
-            "            user_turn_strategies=UserTurnStrategies(\n"
-            "                stop=[\n"
-            "                    TurnAnalyzerUserTurnStopStrategy(\n"
-            "                        turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams())\n"
-            "                    )\n"
-            "                ],\n"
-            "            )\n"
-            "        ),\n"
-            "    )"
-        )
+        logger.warning(f"{self}: `turn_analyzer` in base input transport is deprecated.")
 
         await self._user_turn_controller.update_strategies(ExternalUserTurnStrategies())
 
@@ -516,13 +668,53 @@ class LLMUserAggregator(LLMContextAggregator):
             )
         )
 
+    async def _internal_queue_frame(
+        self,
+        frame: Frame,
+        direction: FrameDirection = FrameDirection.DOWNSTREAM,
+        callback: Optional[FrameCallback] = None,
+    ):
+        """Queues the given frame to ourselves."""
+        self._self_queued_frames.add(frame.id)
+        await self.queue_frame(frame, direction, callback)
+
+    async def _queued_broadcast_frame(self, frame_cls: Type[Frame], **kwargs):
+        """Broadcasts a frame upstream and queues it for internal processing.
+
+        Queues the frame so it flows through `process_frame` and is handled
+        internally (e.g. by the `UserTurnController`). The upstream frame is
+        pushed directly.
+
+        Args:
+            frame_cls: The class of the frame to be broadcasted.
+            **kwargs: Keyword arguments to be passed to the frame's constructor.
+
+        """
+        await self._internal_queue_frame(frame_cls(**kwargs))
+        await self.push_frame(frame_cls(**kwargs), FrameDirection.UPSTREAM)
+
     async def _on_push_frame(
         self, controller, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM
     ):
-        await self.push_frame(frame, direction)
+        await self._internal_queue_frame(frame, direction)
 
     async def _on_broadcast_frame(self, controller, frame_cls: Type[Frame], **kwargs):
-        await self.broadcast_frame(frame_cls, **kwargs)
+        await self._queued_broadcast_frame(frame_cls, **kwargs)
+
+    async def _on_vad_speech_started(self, controller):
+        await self._queued_broadcast_frame(
+            VADUserStartedSpeakingFrame,
+            start_secs=controller._vad_analyzer.params.start_secs,
+        )
+
+    async def _on_vad_speech_stopped(self, controller):
+        await self._queued_broadcast_frame(
+            VADUserStoppedSpeakingFrame,
+            stop_secs=controller._vad_analyzer.params.stop_secs,
+        )
+
+    async def _on_vad_speech_activity(self, controller):
+        await self._queued_broadcast_frame(UserSpeakingFrame)
 
     async def _on_user_turn_started(
         self,
@@ -530,15 +722,17 @@ class LLMUserAggregator(LLMContextAggregator):
         strategy: BaseUserTurnStartStrategy,
         params: UserTurnStartedParams,
     ):
-        logger.debug(f"{self}: User started speaking (user turn start strategy: {strategy})")
+        logger.debug(f"{self}: User started speaking (strategy: {strategy})")
 
         self._user_turn_start_timestamp = time_now_iso8601()
 
         if params.enable_user_speaking_frames:
             await self.broadcast_frame(UserStartedSpeakingFrame)
 
+        await self._user_idle_controller.process_frame(UserStartedSpeakingFrame())
+
         if params.enable_interruptions and self._allow_interruptions:
-            await self.push_interruption_task_frame_and_wait()
+            await self.broadcast_interruption()
 
         await self._call_event_handler("on_user_turn_started", strategy)
 
@@ -548,23 +742,47 @@ class LLMUserAggregator(LLMContextAggregator):
         strategy: BaseUserTurnStopStrategy,
         params: UserTurnStoppedParams,
     ):
-        logger.debug(f"{self}: User stopped speaking (user turn stop strategy: {strategy})")
+        logger.debug(f"{self}: User stopped speaking (strategy: {strategy})")
 
         if params.enable_user_speaking_frames:
             await self.broadcast_frame(UserStoppedSpeakingFrame)
 
-        # Always push context frame.
-        aggregation = await self.push_aggregation()
+        await self._user_idle_controller.process_frame(UserStoppedSpeakingFrame())
 
-        message = UserTurnStoppedMessage(
-            content=aggregation, timestamp=self._user_turn_start_timestamp
-        )
-        await self._call_event_handler("on_user_turn_stopped", strategy, message)
-        self._user_turn_start_timestamp = ""
+        await self._maybe_emit_user_turn_stopped(strategy)
+
+    async def _on_reset_aggregation(
+        self, controller: UserTurnController, strategy: BaseUserTurnStartStrategy
+    ):
+        logger.debug(f"{self}: Resetting aggregation (strategy: {strategy})")
+        await self.reset()
 
     async def _on_user_turn_stop_timeout(self, controller):
         await self._call_event_handler("on_user_turn_stop_timeout")
 
+    async def _on_user_turn_idle(self, controller):
+        await self._call_event_handler("on_user_turn_idle")
+
+    async def _maybe_emit_user_turn_stopped(
+        self,
+        strategy: Optional[BaseUserTurnStopStrategy] = None,
+        on_session_end: bool = False,
+    ):
+        """Maybe emit user turn stopped event.
+
+        Args:
+            strategy: The strategy that triggered the turn stop.
+            on_session_end: If True, only emit if there's unemitted content
+                (avoids duplicate events when session ends).
+        """
+        aggregation = await self.push_aggregation()
+        if not on_session_end or aggregation:
+            message = UserTurnStoppedMessage(
+                content=aggregation, timestamp=self._user_turn_start_timestamp
+            )
+            await self._call_event_handler("on_user_turn_stopped", strategy, message)
+            self._user_turn_start_timestamp = ""
+
 
 class LLMAssistantAggregator(LLMContextAggregator):
     """Assistant LLM aggregator that processes bot responses and function calls.
@@ -585,6 +803,7 @@ class LLMAssistantAggregator(LLMContextAggregator):
     - on_assistant_turn_started: Called when the assistant turn starts
     - on_assistant_turn_stopped: Called when the assistant turn ends
     - on_assistant_thought: Called when an assistant thought is available
+    - on_summary_applied: Called when a context summarization is applied
 
     Example::
 
@@ -600,6 +819,10 @@ class LLMAssistantAggregator(LLMContextAggregator):
         async def on_assistant_thought(aggregator, message: AssistantThoughtMessage):
             ...
 
+        @aggregator.event_handler("on_summary_applied")
+        async def on_summary_applied(aggregator, summarizer, event: SummaryAppliedEvent):
+            ...
+
     """
 
     def __init__(
@@ -639,8 +862,8 @@ class LLMAssistantAggregator(LLMContextAggregator):
                     DeprecationWarning,
                 )
 
-        self._started = 0
         self._function_calls_in_progress: Dict[str, Optional[FunctionCallInProgressFrame]] = {}
+        self._function_calls_image_results: Dict[str, UserImageRawFrame] = {}
         self._context_updated_tasks: Set[asyncio.Task] = set()
 
         self._assistant_turn_start_timestamp = ""
@@ -650,9 +873,24 @@ class LLMAssistantAggregator(LLMContextAggregator):
         self._thought_aggregation: List[TextPartForConcatenation] = []
         self._thought_start_time: str = ""
 
+        # Context summarization — always create the summarizer so that manually
+        # pushed LLMSummarizeContextFrame frames are always handled.
+        # Auto-triggering based on thresholds is only enabled when
+        # enable_auto_context_summarization is True.
+        self._summarizer: Optional[LLMContextSummarizer] = LLMContextSummarizer(
+            context=self._context,
+            config=self._params.auto_context_summarization_config,
+            auto_trigger=self._params.enable_auto_context_summarization,
+        )
+        self._summarizer.add_event_handler(
+            "on_request_summarization", self._on_request_summarization
+        )
+        self._summarizer.add_event_handler("on_summary_applied", self._on_summary_applied)
+
         self._register_event_handler("on_assistant_turn_started")
         self._register_event_handler("on_assistant_turn_stopped")
         self._register_event_handler("on_assistant_thought")
+        self._register_event_handler("on_summary_applied")
 
     @property
     def has_function_calls_in_progress(self) -> bool:
@@ -683,9 +921,19 @@ class LLMAssistantAggregator(LLMContextAggregator):
         """
         await super().process_frame(frame, direction)
 
-        if isinstance(frame, InterruptionFrame):
+        if isinstance(frame, StartFrame):
+            # Push StartFrame before start(), because we want StartFrame to be
+            # processed by every processor before any other frame is processed.
+            await self.push_frame(frame, direction)
+            await self._start(frame)
+        elif isinstance(frame, InterruptionFrame):
             await self._handle_interruptions(frame)
             await self.push_frame(frame, direction)
+        elif isinstance(frame, (EndFrame, CancelFrame)):
+            await self._handle_end_or_cancel(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, LLMAssistantPushAggregationFrame):
+            await self.push_aggregation()
         elif isinstance(frame, LLMFullResponseStartFrame):
             await self._handle_llm_start(frame)
         elif isinstance(frame, LLMFullResponseEndFrame):
@@ -723,6 +971,14 @@ class LLMAssistantAggregator(LLMContextAggregator):
         else:
             await self.push_frame(frame, direction)
 
+        # Pass frames to summarizer for monitoring
+        if self._summarizer:
+            await self._summarizer.process_frame(frame)
+
+    async def _start(self, frame: StartFrame):
+        if self._summarizer:
+            await self._summarizer.setup(self.task_manager)
+
     async def push_aggregation(self) -> str:
         """Push the current assistant aggregation with timestamp."""
         if not self._aggregation:
@@ -757,9 +1013,13 @@ class LLMAssistantAggregator(LLMContextAggregator):
 
     async def _handle_interruptions(self, frame: InterruptionFrame):
         await self._trigger_assistant_turn_stopped()
-        self._started = 0
         await self.reset()
 
+    async def _handle_end_or_cancel(self, frame: Frame):
+        await self._trigger_assistant_turn_stopped()
+        if self._summarizer:
+            await self._summarizer.cleanup()
+
     async def _handle_function_calls_started(self, frame: FunctionCallsStartedFrame):
         function_names = [f"{f.function_name}:{f.tool_call_id}" for f in frame.function_calls]
         logger.debug(f"{self} FunctionCallsStartedFrame: {function_names}")
@@ -780,7 +1040,7 @@ class LLMAssistantAggregator(LLMContextAggregator):
                         "id": frame.tool_call_id,
                         "function": {
                             "name": frame.function_name,
-                            "arguments": json.dumps(frame.arguments),
+                            "arguments": json.dumps(frame.arguments, ensure_ascii=False),
                         },
                         "type": "function",
                     }
@@ -813,13 +1073,22 @@ class LLMAssistantAggregator(LLMContextAggregator):
 
         # Update context with the function call result
         if frame.result:
-            result = json.dumps(frame.result)
+            result = json.dumps(frame.result, ensure_ascii=False)
             self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
         else:
             self._update_function_call_result(frame.function_name, frame.tool_call_id, "COMPLETED")
 
         run_llm = False
 
+        # Append any images that were generated by function calls.
+        if frame.tool_call_id in self._function_calls_image_results:
+            image_frame = self._function_calls_image_results[frame.tool_call_id]
+
+            del self._function_calls_image_results[frame.tool_call_id]
+
+            # If an image frame has been added to the context, let's run inference.
+            run_llm = await self._maybe_append_image_to_context(image_frame)
+
         # Run inference if the function call result requires it.
         if frame.result:
             if properties and properties.run_llm is not None:
@@ -848,39 +1117,32 @@ class LLMAssistantAggregator(LLMContextAggregator):
         logger.debug(
             f"{self} FunctionCallCancelFrame: [{frame.function_name}:{frame.tool_call_id}]"
         )
-        if frame.tool_call_id not in self._function_calls_in_progress:
-            return
-
-        if self._function_calls_in_progress[frame.tool_call_id].cancel_on_interruption:
+        function_call = self._function_calls_in_progress.get(frame.tool_call_id)
+        if function_call and function_call.cancel_on_interruption:
             # Update context with the function call cancellation
             self._update_function_call_result(frame.function_name, frame.tool_call_id, "CANCELLED")
             del self._function_calls_in_progress[frame.tool_call_id]
 
-    def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any):
-        for message in self._context.get_messages():
-            if (
-                not isinstance(message, LLMSpecificMessage)
-                and message["role"] == "tool"
-                and message["tool_call_id"]
-                and message["tool_call_id"] == tool_call_id
-            ):
-                message["content"] = result
-
     async def _handle_user_image_frame(self, frame: UserImageRawFrame):
-        if not frame.append_to_context:
-            return
+        image_appended = False
 
-        logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})")
+        # Check if this image is a result of a function call.
+        if (
+            frame.request
+            and frame.request.tool_call_id
+            and frame.request.tool_call_id in self._function_calls_in_progress
+        ):
+            self._function_calls_image_results[frame.request.tool_call_id] = frame
 
-        await self._context.add_image_frame_message(
-            format=frame.format,
-            size=frame.size,
-            image=frame.image,
-            text=frame.text,
-        )
+            # Call the result_callback if provided. This signals that the image
+            # has been retrieved and the function call can now complete.
+            if frame.request.result_callback:
+                await frame.request.result_callback(None)
+        else:
+            image_appended = await self._maybe_append_image_to_context(frame)
 
-        await self._trigger_assistant_turn_stopped()
-        await self.push_context_frame(FrameDirection.UPSTREAM)
+        if image_appended:
+            await self.push_context_frame(FrameDirection.UPSTREAM)
 
     async def _handle_assistant_image_frame(self, frame: AssistantImageRawFrame):
         logger.debug(f"{self} Appending AssistantImageRawFrame to LLM context (size: {frame.size})")
@@ -901,15 +1163,17 @@ class LLMAssistantAggregator(LLMContextAggregator):
             )
 
     async def _handle_llm_start(self, _: LLMFullResponseStartFrame):
-        self._started += 1
         await self._trigger_assistant_turn_started()
 
     async def _handle_llm_end(self, _: LLMFullResponseEndFrame):
-        self._started -= 1
         await self._trigger_assistant_turn_stopped()
 
     async def _handle_text(self, frame: TextFrame):
-        if not self._started or not frame.append_to_context:
+        # Skip TextFrame types not intended to build the assistant context
+        if isinstance(frame, (TranscriptionFrame, TranslationFrame, InterimTranscriptionFrame)):
+            return
+
+        if not frame.append_to_context:
             return
 
         # Make sure we really have text (spaces count, too!)
@@ -923,18 +1187,12 @@ class LLMAssistantAggregator(LLMContextAggregator):
         )
 
     async def _handle_thought_start(self, frame: LLMThoughtStartFrame):
-        if not self._started:
-            return
-
         await self._reset_thought_aggregation()
         self._thought_append_to_context = frame.append_to_context
         self._thought_llm = frame.llm
         self._thought_start_time = time_now_iso8601()
 
     async def _handle_thought_text(self, frame: LLMThoughtTextFrame):
-        if not self._started:
-            return
-
         # Make sure we really have text (spaces count, too!)
         if len(frame.text) == 0:
             return
@@ -946,11 +1204,7 @@ class LLMAssistantAggregator(LLMContextAggregator):
         )
 
     async def _handle_thought_end(self, frame: LLMThoughtEndFrame):
-        if not self._started:
-            return
-
         thought = concatenate_aggregated_text(self._thought_aggregation)
-        await self._reset_thought_aggregation()
 
         if self._thought_append_to_context:
             llm = self._thought_llm
@@ -966,8 +1220,36 @@ class LLMAssistantAggregator(LLMContextAggregator):
             )
 
         message = AssistantThoughtMessage(content=thought, timestamp=self._thought_start_time)
+
+        await self._reset_thought_aggregation()
+
         await self._call_event_handler("on_assistant_thought", message)
 
+    async def _maybe_append_image_to_context(self, frame: UserImageRawFrame) -> bool:
+        if not frame.append_to_context:
+            return False
+
+        logger.debug(f"{self} Appending UserImageRawFrame to LLM context (size: {frame.size})")
+
+        await self._context.add_image_frame_message(
+            format=frame.format,
+            size=frame.size,
+            image=frame.image,
+            text=frame.text,
+        )
+
+        return True
+
+    def _update_function_call_result(self, function_name: str, tool_call_id: str, result: Any):
+        for message in self._context.get_messages():
+            if (
+                not isinstance(message, LLMSpecificMessage)
+                and message["role"] == "tool"
+                and message["tool_call_id"]
+                and message["tool_call_id"] == tool_call_id
+            ):
+                message["content"] = result
+
     def _context_updated_task_finished(self, task: asyncio.Task):
         self._context_updated_tasks.discard(task)
 
@@ -979,13 +1261,66 @@ class LLMAssistantAggregator(LLMContextAggregator):
     async def _trigger_assistant_turn_stopped(self):
         aggregation = await self.push_aggregation()
         if aggregation:
+            # Strip turn completion markers from the transcript
+            content = self._maybe_strip_turn_completion_markers(aggregation)
             message = AssistantTurnStoppedMessage(
-                content=aggregation, timestamp=self._assistant_turn_start_timestamp
+                content=content, timestamp=self._assistant_turn_start_timestamp
             )
             await self._call_event_handler("on_assistant_turn_stopped", message)
 
             self._assistant_turn_start_timestamp = ""
 
+    def _maybe_strip_turn_completion_markers(self, text: str) -> str:
+        """Strip turn completion markers from assistant transcript.
+
+        These markers (✓, ○, ◐) are used internally for turn completion
+        detection and shouldn't appear in the final transcript.
+        """
+        from pipecat.turns.user_turn_completion_mixin import (
+            USER_TURN_COMPLETE_MARKER,
+            USER_TURN_INCOMPLETE_LONG_MARKER,
+            USER_TURN_INCOMPLETE_SHORT_MARKER,
+        )
+
+        marker_found = False
+        for marker in (
+            USER_TURN_COMPLETE_MARKER,
+            USER_TURN_INCOMPLETE_SHORT_MARKER,
+            USER_TURN_INCOMPLETE_LONG_MARKER,
+        ):
+            if marker in text:
+                text = text.replace(marker, "")
+                marker_found = True
+
+        # Only strip whitespace if we removed a marker
+        return text.strip() if marker_found else text
+
+    async def _on_request_summarization(
+        self, summarizer: LLMContextSummarizer, frame: LLMContextSummaryRequestFrame
+    ):
+        """Handle summarization request from the summarizer.
+
+        Push the request frame UPSTREAM to the LLM service for processing.
+
+        Args:
+            summarizer: The summarizer that generated the request.
+            frame: The summarization request frame to broadcast.
+        """
+        await self.push_frame(frame, FrameDirection.UPSTREAM)
+
+    async def _on_summary_applied(
+        self, summarizer: LLMContextSummarizer, event: SummaryAppliedEvent
+    ):
+        """Handle summary applied event from the summarizer.
+
+        Forwards the event to any registered `on_summary_applied` handlers.
+
+        Args:
+            summarizer: The summarizer that applied the summary.
+            event: The summary applied event.
+        """
+        await self._call_event_handler("on_summary_applied", summarizer, event)
+
 
 class LLMContextAggregatorPair:
     """Pair of LLM context aggregators for updating context with user and assistant messages."""
@@ -994,8 +1329,8 @@ class LLMContextAggregatorPair:
         self,
         context: LLMContext,
         *,
-        user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
-        assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
+        user_params: Optional[LLMUserAggregatorParams] = None,
+        assistant_params: Optional[LLMAssistantAggregatorParams] = None,
     ):
         """Initialize the LLM context aggregator pair.
 
@@ -1004,6 +1339,8 @@ class LLMContextAggregatorPair:
             user_params: Parameters for the user context aggregator.
             assistant_params: Parameters for the assistant context aggregator.
         """
+        user_params = user_params or LLMUserAggregatorParams()
+        assistant_params = assistant_params or LLMAssistantAggregatorParams()
         self._user = LLMUserAggregator(context, params=user_params)
         self._assistant = LLMAssistantAggregator(context, params=assistant_params)
 
@@ -1022,3 +1359,15 @@ class LLMContextAggregatorPair:
             The assistant context aggregator instance.
         """
         return self._assistant
+
+    def __iter__(self):
+        """Allow tuple unpacking of the aggregator pair.
+
+        This enables both usage patterns::
+            pair = LLMContextAggregatorPair(context)  # Returns the instance
+            user, assistant = LLMContextAggregatorPair(context)  # Unpacks into tuple
+
+        Yields:
+            The user aggregator, then the assistant aggregator.
+        """
+        return iter((self._user, self._assistant))

src/pipecat/processors/aggregators/openai_llm_context.py

@@ -34,7 +34,6 @@ from PIL import Image
 from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.frames.frames import AudioRawFrame, Frame
-from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 # JSON custom encoder to handle bytes arrays so that we can log contexts
 # with images to the console.

src/pipecat/processors/audio/audio_buffer_processor.py

@@ -11,7 +11,6 @@ of audio from both user input and bot output sources, with support for various a
 configurations and event-driven processing.
 """
 
-import time
 from typing import Optional
 
 from pipecat.audio.utils import create_stream_resampler, interleave_stereo_audio, mix_audio
@@ -104,10 +103,6 @@ class AudioBufferProcessor(FrameProcessor):
         self._user_turn_audio_buffer = bytearray()
         self._bot_turn_audio_buffer = bytearray()
 
-        # Intermittent (non continous user stream variables)
-        self._last_user_frame_at = 0
-        self._last_bot_frame_at = 0
-
         self._recording = False
 
         self._input_resampler = create_stream_resampler()
@@ -211,23 +206,31 @@ class AudioBufferProcessor(FrameProcessor):
         """Process audio frames for recording."""
         resampled = None
         if isinstance(frame, InputAudioRawFrame):
-            # Add silence if we need to.
-            silence = self._compute_silence(self._last_user_frame_at)
-            self._user_audio_buffer.extend(silence)
-            # Add user audio.
             resampled = await self._resample_input_audio(frame)
-            self._user_audio_buffer.extend(resampled)
-            # Save time of frame so we can compute silence.
-            self._last_user_frame_at = time.time()
+            # Ignoring in case we don't have audio
+            if len(resampled) > 0:
+                # Sync bot buffer to current user position before adding user audio.
+                # We sync BEFORE extending to align both buffers at the same starting timestamp.
+                # For example, user buffer is at 100 bytes, and you receive 20 bytes of new audio
+                #  - Bot buffer sees User is at 100. Bot pads itself to 100.
+                #  - User buffer adds 20. User is now at 120.
+                #  - Outcome: At index 100-120, we have User Audio and (potentially) Bot Audio or silence. They are aligned
+                # This gives the opportunity to the bot to send audio.
+                #
+                # If we synced AFTER, we'd pad the bot buffer with silence for the same
+                # window we just gave to the user, effectively "overwriting" that time slot
+                # with silence and causing the bot's audio to flicker or cut out.
+                self._sync_buffer_to_position(self._bot_audio_buffer, len(self._user_audio_buffer))
+                # Add user audio.
+                self._user_audio_buffer.extend(resampled)
         elif self._recording and isinstance(frame, OutputAudioRawFrame):
-            # Add silence if we need to.
-            silence = self._compute_silence(self._last_bot_frame_at)
-            self._bot_audio_buffer.extend(silence)
-            # Add bot audio.
             resampled = await self._resample_output_audio(frame)
-            self._bot_audio_buffer.extend(resampled)
-            # Save time of frame so we can compute silence.
-            self._last_bot_frame_at = time.time()
+            # Ignoring in case we don't have audio
+            if len(resampled) > 0:
+                # Sync user buffer to current bot position before adding bot audio
+                self._sync_buffer_to_position(self._user_audio_buffer, len(self._bot_audio_buffer))
+                # Add bot audio.
+                self._bot_audio_buffer.extend(resampled)
 
         if self._buffer_size > 0 and (
             len(self._user_audio_buffer) >= self._buffer_size
@@ -240,6 +243,21 @@ class AudioBufferProcessor(FrameProcessor):
         if self._enable_turn_audio:
             await self._process_turn_recording(frame, resampled)
 
+    def _sync_buffer_to_position(self, buffer: bytearray, target_position: int):
+        """Pad buffer with silence if it's behind the target position.
+
+        This ensures both buffers stay synchronized by padding the lagging
+        buffer before new audio is added to the other buffer.
+
+        Args:
+            buffer: The buffer to potentially pad.
+            target_position: The position (in bytes) the buffer should reach.
+        """
+        current_len = len(buffer)
+        if current_len < target_position:
+            silence_needed = target_position - current_len
+            buffer.extend(b"\x00" * silence_needed)
+
     async def _process_turn_recording(self, frame: Frame, resampled_audio: Optional[bytes] = None):
         """Process frames for turn-based audio recording."""
         if isinstance(frame, UserStartedSpeakingFrame):
@@ -281,8 +299,8 @@ class AudioBufferProcessor(FrameProcessor):
         if len(self._user_audio_buffer) == 0 and len(self._bot_audio_buffer) == 0:
             return
 
+        # Final alignment before we send the audio
         self._align_track_buffers()
-        flush_time = time.time()
 
         # Call original handler with merged audio
         merged_audio = self.merge_audio_buffers()
@@ -299,9 +317,6 @@ class AudioBufferProcessor(FrameProcessor):
             self._num_channels,
         )
 
-        self._last_user_frame_at = flush_time
-        self._last_bot_frame_at = flush_time
-
     def _buffer_has_audio(self, buffer: bytearray) -> bool:
         """Check if a buffer contains audio data."""
         return buffer is not None and len(buffer) > 0
@@ -309,8 +324,6 @@ class AudioBufferProcessor(FrameProcessor):
     def _reset_recording(self):
         """Reset recording state and buffers."""
         self._reset_all_audio_buffers()
-        self._last_user_frame_at = time.time()
-        self._last_bot_frame_at = time.time()
 
     def _reset_all_audio_buffers(self):
         """Reset all audio buffers to empty state."""
@@ -336,11 +349,9 @@ class AudioBufferProcessor(FrameProcessor):
 
         target_len = max(user_len, bot_len)
         if user_len < target_len:
-            self._user_audio_buffer.extend(b"\x00" * (target_len - user_len))
-            self._last_user_frame_at = max(self._last_user_frame_at, self._last_bot_frame_at)
+            self._sync_buffer_to_position(self._user_audio_buffer, target_len)
         if bot_len < target_len:
-            self._bot_audio_buffer.extend(b"\x00" * (target_len - bot_len))
-            self._last_bot_frame_at = max(self._last_bot_frame_at, self._last_user_frame_at)
+            self._sync_buffer_to_position(self._bot_audio_buffer, target_len)
 
     async def _resample_input_audio(self, frame: InputAudioRawFrame) -> bytes:
         """Resample audio frame to the target sample rate."""
@@ -353,14 +364,3 @@ class AudioBufferProcessor(FrameProcessor):
         return await self._output_resampler.resample(
             frame.audio, frame.sample_rate, self._sample_rate
         )
-
-    def _compute_silence(self, from_time: float) -> bytes:
-        """Compute silence to insert based on time gap."""
-        quiet_time = time.time() - from_time
-        # We should get audio frames very frequently. We introduce silence only
-        # if there's a big enough gap of 1s.
-        if from_time == 0 or quiet_time < 1.0:
-            return b""
-        num_bytes = int(quiet_time * self._sample_rate) * 2
-        silence = b"\x00" * num_bytes
-        return silence

src/pipecat/processors/audio/vad_processor.py

@@ -0,0 +1,108 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Voice Activity Detection processor for detecting speech in audio streams.
+
+This module provides a VADProcessor that wraps a VADController to process
+audio frames and push VAD-related frames into the pipeline.
+"""
+
+from typing import Type
+
+from loguru import logger
+
+from pipecat.audio.vad.vad_analyzer import VADAnalyzer
+from pipecat.audio.vad.vad_controller import VADController
+from pipecat.frames.frames import (
+    Frame,
+    UserSpeakingFrame,
+    VADUserStartedSpeakingFrame,
+    VADUserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+
+
+class VADProcessor(FrameProcessor):
+    """Processes audio frames through voice activity detection.
+
+    This processor wraps a VADController to detect speech in audio streams
+    and push VAD frames into the pipeline:
+
+    - ``VADUserStartedSpeakingFrame``: Pushed when speech begins.
+    - ``VADUserStoppedSpeakingFrame``: Pushed when speech ends.
+    - ``UserSpeakingFrame``: Pushed periodically while speech is detected.
+
+    Example::
+
+        vad_processor = VADProcessor(vad_analyzer=SileroVADAnalyzer())
+    """
+
+    def __init__(
+        self,
+        *,
+        vad_analyzer: VADAnalyzer,
+        speech_activity_period: float = 0.2,
+        **kwargs,
+    ):
+        """Initialize the VAD processor.
+
+        Args:
+            vad_analyzer: The VADAnalyzer instance for processing audio.
+            speech_activity_period: Minimum interval in seconds between
+                UserSpeakingFrame pushes. Defaults to 0.2.
+            **kwargs: Additional arguments passed to parent class.
+        """
+        super().__init__(**kwargs)
+        self._vad_controller = VADController(
+            vad_analyzer, speech_activity_period=speech_activity_period
+        )
+
+        # Push VAD frames when speech events are detected
+        @self._vad_controller.event_handler("on_speech_started")
+        async def on_speech_started(_controller):
+            logger.debug(f"{self}: User started speaking")
+            await self.broadcast_frame(
+                VADUserStartedSpeakingFrame,
+                start_secs=_controller._vad_analyzer.params.start_secs,
+            )
+
+        @self._vad_controller.event_handler("on_speech_stopped")
+        async def on_speech_stopped(_controller):
+            logger.debug(f"{self}: User stopped speaking")
+            await self.broadcast_frame(
+                VADUserStoppedSpeakingFrame,
+                stop_secs=_controller._vad_analyzer.params.stop_secs,
+            )
+
+        @self._vad_controller.event_handler("on_speech_activity")
+        async def on_speech_activity(_controller):
+            await self.broadcast_frame(UserSpeakingFrame)
+
+        # Wire up frame pushing from controller to processor
+        @self._vad_controller.event_handler("on_push_frame")
+        async def on_push_frame(_controller, frame: Frame, direction: FrameDirection):
+            await self.push_frame(frame, direction)
+
+        @self._vad_controller.event_handler("on_broadcast_frame")
+        async def on_broadcast_frame(_controller, frame_cls: Type[Frame], **kwargs):
+            await self.broadcast_frame(frame_cls, **kwargs)
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process a frame through VAD and forward it.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame flow in the pipeline.
+        """
+        await super().process_frame(frame, direction)
+
+        # Forward the frame first, then let VAD controller process. This ensures:
+        # 1. StartFrame reaches downstream before SpeechControlParamsFrame is broadcast
+        # 2. Audio flows through immediately while VAD detection happens after
+        await self.push_frame(frame, direction)
+
+        # Let the VAD controller handle the frame
+        await self._vad_controller.process_frame(frame)

src/pipecat/processors/filters/function_filter.py

@@ -10,11 +10,13 @@ This module provides a processor that filters frames based on a custom function,
 allowing for flexible frame filtering logic in processing pipelines.
 """
 
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Optional
 
 from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
+FilterType = Callable[[Frame], Awaitable[bool]]
+
 
 class FunctionFilter(FrameProcessor):
     """A frame processor that filters frames using a custom function.
@@ -26,9 +28,10 @@ class FunctionFilter(FrameProcessor):
 
     def __init__(
         self,
-        filter: Callable[[Frame], Awaitable[bool]],
-        direction: FrameDirection = FrameDirection.DOWNSTREAM,
+        filter: FilterType,
+        direction: Optional[FrameDirection] = FrameDirection.DOWNSTREAM,
         filter_system_frames: bool = False,
+        **kwargs,
     ):
         """Initialize the function filter.
 
@@ -36,10 +39,13 @@ class FunctionFilter(FrameProcessor):
             filter: An async function that takes a Frame and returns True if the
                 frame should pass through, False otherwise.
             direction: The direction to apply filtering. Only frames moving in
-                this direction will be filtered. Defaults to DOWNSTREAM.
+                this direction will be filtered; frames in the other direction
+                pass through unfiltered. If None, frames in both directions
+                are filtered. Defaults to DOWNSTREAM.
             filter_system_frames: Whether to filter system frames. Defaults to False.
+            **kwargs: Additional arguments passed to parent class.
         """
-        super().__init__()
+        super().__init__(**kwargs)
         self._filter = filter
         self._direction = direction
         self._filter_system_frames = filter_system_frames
@@ -51,7 +57,7 @@ class FunctionFilter(FrameProcessor):
     def _should_passthrough_frame(self, frame, direction):
         """Check if a frame should pass through without filtering."""
         # Always passthrough frames in the wrong direction
-        if direction != self._direction:
+        if self._direction and direction != self._direction:
             return True
 
         # Always passthrough lifecycle frames

src/pipecat/processors/filters/wake_check_filter.py

@@ -6,6 +6,9 @@
 
 """Wake phrase detection filter for Pipecat transcription processing.
 
+.. deprecated:: 0.0.106
+    Use :class:`~pipecat.turns.user_start.WakePhraseUserTurnStartStrategy` instead.
+
 This module provides a frame processor that filters transcription frames,
 only allowing them through after wake phrases have been detected. Includes
 keepalive functionality to maintain conversation flow after wake detection.
@@ -13,18 +16,24 @@ keepalive functionality to maintain conversation flow after wake detection.
 
 import re
 import time
+import warnings
 from enum import Enum
 from typing import List
 
 from loguru import logger
 
-from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame
+from pipecat.frames.frames import Frame, TranscriptionFrame
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 
 
 class WakeCheckFilter(FrameProcessor):
     """Frame processor that filters transcription frames based on wake phrase detection.
 
+    .. deprecated:: 0.0.106
+        Use :class:`~pipecat.turns.user_start.WakePhraseUserTurnStartStrategy` instead,
+        which integrates with the user turn strategy system and supports configurable
+        timeouts and single-activation mode.
+
     This filter monitors transcription frames for configured wake phrases and only
     passes frames through after a wake phrase has been detected. Maintains a
     keepalive timeout to allow continued conversation after wake detection.
@@ -65,12 +74,21 @@ class WakeCheckFilter(FrameProcessor):
     def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3):
         """Initialize the wake phrase filter.
 
+        .. deprecated:: 0.0.106
+            Use :class:`~pipecat.turns.user_start.WakePhraseUserTurnStartStrategy` instead.
+
         Args:
             wake_phrases: List of wake phrases to detect in transcriptions.
             keepalive_timeout: Duration in seconds to keep passing frames after
                 wake detection. Defaults to 3 seconds.
         """
         super().__init__()
+        warnings.warn(
+            "WakeCheckFilter is deprecated since v0.0.106. "
+            "Use WakePhraseUserTurnStartStrategy instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
         self._participant_states = {}
         self._keepalive_timeout = keepalive_timeout
         self._wake_patterns = []

src/pipecat/processors/frame_processor.py

@@ -12,6 +12,7 @@ management, and frame flow control mechanisms.
 """
 
 import asyncio
+import dataclasses
 import traceback
 from dataclasses import dataclass
 from enum import Enum
@@ -40,7 +41,6 @@ from pipecat.frames.frames import (
     FrameProcessorResumeFrame,
     FrameProcessorResumeUrgentFrame,
     InterruptionFrame,
-    InterruptionTaskFrame,
     StartFrame,
     SystemFrame,
     UninterruptibleFrame,
@@ -239,14 +239,6 @@ class FrameProcessor(BaseObject):
         self.__process_frame_task: Optional[asyncio.Task] = None
         self.__process_current_frame: Optional[Frame] = None
 
-        # To interrupt a pipeline, we push an `InterruptionTaskFrame` upstream.
-        # Then we wait for the corresponding `InterruptionFrame` to travel from
-        # the start of the pipeline back to the processor that sent the
-        # `InterruptionTaskFrame`. This wait is handled using the following
-        # event.
-        self._wait_for_interruption = False
-        self._wait_interruption_event = asyncio.Event()
-
         # Frame processor events.
         self._register_event_handler("on_before_process_frame", sync=True)
         self._register_event_handler("on_after_process_frame", sync=True)
@@ -332,7 +324,7 @@ class FrameProcessor(BaseObject):
             warnings.simplefilter("always")
             warnings.warn(
                 "`FrameProcessor.interruptions_allowed` is deprecated. "
-                "Use  `LLMUserAggregator`'s new `user_mute_strategies` parameter instead.",
+                "Use `LLMUserAggregator`'s new `user_mute_strategies` parameter instead.",
                 DeprecationWarning,
                 stacklevel=2,
             )
@@ -420,27 +412,49 @@ class FrameProcessor(BaseObject):
         """
         self._metrics.set_core_metrics_data(data)
 
-    async def start_ttfb_metrics(self):
-        """Start time-to-first-byte metrics collection."""
-        if self.can_generate_metrics() and self.metrics_enabled:
-            await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb)
+    async def start_ttfb_metrics(self, *, start_time: Optional[float] = None):
+        """Start time-to-first-byte metrics collection.
 
-    async def stop_ttfb_metrics(self):
-        """Stop time-to-first-byte metrics collection and push results."""
+        Args:
+            start_time: Optional timestamp to use as the start time. If None,
+                uses the current time.
+        """
         if self.can_generate_metrics() and self.metrics_enabled:
-            frame = await self._metrics.stop_ttfb_metrics()
+            await self._metrics.start_ttfb_metrics(
+                start_time=start_time, report_only_initial_ttfb=self._report_only_initial_ttfb
+            )
+
+    async def stop_ttfb_metrics(self, *, end_time: Optional[float] = None):
+        """Stop time-to-first-byte metrics collection and push results.
+
+        Args:
+            end_time: Optional timestamp to use as the end time. If None, uses
+                the current time.
+        """
+        if self.can_generate_metrics() and self.metrics_enabled:
+            frame = await self._metrics.stop_ttfb_metrics(end_time=end_time)
             if frame:
                 await self.push_frame(frame)
 
-    async def start_processing_metrics(self):
-        """Start processing metrics collection."""
-        if self.can_generate_metrics() and self.metrics_enabled:
-            await self._metrics.start_processing_metrics()
+    async def start_processing_metrics(self, *, start_time: Optional[float] = None):
+        """Start processing metrics collection.
 
-    async def stop_processing_metrics(self):
-        """Stop processing metrics collection and push results."""
+        Args:
+            start_time: Optional timestamp to use as the start time. If None,
+                uses the current time.
+        """
         if self.can_generate_metrics() and self.metrics_enabled:
-            frame = await self._metrics.stop_processing_metrics()
+            await self._metrics.start_processing_metrics(start_time=start_time)
+
+    async def stop_processing_metrics(self, *, end_time: Optional[float] = None):
+        """Stop processing metrics collection and push results.
+
+        Args:
+            end_time: Optional timestamp to use as the end time. If None, uses
+                the current time.
+        """
+        if self.can_generate_metrics() and self.metrics_enabled:
+            frame = await self._metrics.stop_processing_metrics(end_time=end_time)
             if frame:
                 await self.push_frame(frame)
 
@@ -466,10 +480,23 @@ class FrameProcessor(BaseObject):
             if frame:
                 await self.push_frame(frame)
 
+    async def start_text_aggregation_metrics(self):
+        """Start text aggregation time metrics collection."""
+        if self.can_generate_metrics() and self.metrics_enabled:
+            await self._metrics.start_text_aggregation_metrics()
+
+    async def stop_text_aggregation_metrics(self):
+        """Stop text aggregation time metrics collection and push results."""
+        if self.can_generate_metrics() and self.metrics_enabled:
+            frame = await self._metrics.stop_text_aggregation_metrics()
+            if frame:
+                await self.push_frame(frame)
+
     async def stop_all_metrics(self):
         """Stop all active metrics collection."""
         await self.stop_ttfb_metrics()
         await self.stop_processing_metrics()
+        await self.stop_text_aggregation_metrics()
 
     def create_task(self, coroutine: Coroutine, name: Optional[str] = None) -> asyncio.Task:
         """Create a new task managed by this processor.
@@ -599,14 +626,6 @@ class FrameProcessor(BaseObject):
         if self._cancelling:
             return
 
-        # If we are waiting for an interruption we will bypass all queued system
-        # frames and we will process the frame right away. This is because a
-        # previous system frame might be waiting for the interruption frame and
-        # it's blocking the input task.
-        if self._wait_for_interruption and isinstance(frame, InterruptionFrame):
-            await self.__process_frame(frame, direction, callback)
-            return
-
         if self._enable_direct_mode:
             await self.__process_frame(frame, direction, callback)
         else:
@@ -741,46 +760,85 @@ class FrameProcessor(BaseObject):
 
         await self._call_event_handler("on_after_push_frame", frame)
 
-        # If we are waiting for an interruption and we get an interruption, then
-        # we can unblock `push_interruption_task_frame_and_wait()`.
-        if self._wait_for_interruption and isinstance(frame, InterruptionFrame):
-            self._wait_interruption_event.set()
+    async def broadcast_interruption(self):
+        """Broadcast an `InterruptionFrame` both upstream and downstream."""
+        logger.debug(f"{self}: broadcasting interruption")
+        self.__reset_process_task()
+        await self.stop_all_metrics()
+        await self.broadcast_frame(InterruptionFrame)
 
-    async def push_interruption_task_frame_and_wait(self):
+    async def push_interruption_task_frame_and_wait(self, *, timeout: float = 5.0):
         """Push an interruption task frame upstream and wait for the interruption.
 
-        This function sends an `InterruptionTaskFrame` upstream to the pipeline
-        task and waits to receive the corresponding `InterruptionFrame`. When
-        the function finishes it is guaranteed that the `InterruptionFrame` has
-        been pushed downstream.
+        .. deprecated:: 0.0.104
+            Use :meth:`broadcast_interruption` instead. This method now
+            delegates to ``broadcast_interruption()`` and ignores *timeout*.
         """
-        self._wait_for_interruption = True
+        import warnings
 
-        await self.push_frame(InterruptionTaskFrame(), FrameDirection.UPSTREAM)
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "`FrameProcessor.push_interruption_task_frame_and_wait()` is deprecated. "
+                "Use `FrameProcessor.broadcast_interruption()` instead.",
+                DeprecationWarning,
+                stacklevel=2,
+            )
 
-        # Wait for an `InterruptionFrame` to come to this processor and be
-        # pushed. Take a look at `push_frame()` to see how we first push the
-        # `InterruptionFrame` and then we set the event in order to maintain
-        # frame ordering.
-        await self._wait_interruption_event.wait()
-
-        # Clean the event.
-        self._wait_interruption_event.clear()
-
-        self._wait_for_interruption = False
+        await self.broadcast_interruption()
 
     async def broadcast_frame(self, frame_cls: Type[Frame], **kwargs):
         """Broadcasts a frame of the specified class upstream and downstream.
 
         This method creates two instances of the given frame class using the
-        provided keyword arguments and pushes them upstream and downstream.
+        provided keyword arguments (without deep-copying them) and pushes them
+        upstream and downstream.
 
         Args:
             frame_cls: The class of the frame to be broadcasted.
             **kwargs: Keyword arguments to be passed to the frame's constructor.
         """
-        await self.push_frame(frame_cls(**kwargs))
-        await self.push_frame(frame_cls(**kwargs), FrameDirection.UPSTREAM)
+        downstream_frame = frame_cls(**kwargs)
+        upstream_frame = frame_cls(**kwargs)
+        downstream_frame.broadcast_sibling_id = upstream_frame.id
+        upstream_frame.broadcast_sibling_id = downstream_frame.id
+        await self.push_frame(downstream_frame)
+        await self.push_frame(upstream_frame, FrameDirection.UPSTREAM)
+
+    async def broadcast_frame_instance(self, frame: Frame):
+        """Broadcasts a frame instance upstream and downstream.
+
+        This method creates two new frame instances shallow-copying all fields
+        from the original frame except `id` and `name`, which get fresh values.
+
+        Args:
+            frame: The frame instance to broadcast.
+
+        Note:
+            Prefer using `broadcast_frame()` when possible, as it is more
+            efficient. This method should only be used when you are not the
+            creator of the frame and need to broadcast an existing instance.
+        """
+        frame_cls = type(frame)
+        init_fields = {f.name: getattr(frame, f.name) for f in dataclasses.fields(frame) if f.init}
+        extra_fields = {
+            f.name: getattr(frame, f.name)
+            for f in dataclasses.fields(frame)
+            if not f.init and f.name not in ("id", "name")
+        }
+
+        downstream_frame = frame_cls(**init_fields)
+        for k, v in extra_fields.items():
+            setattr(downstream_frame, k, v)
+
+        upstream_frame = frame_cls(**init_fields)
+        for k, v in extra_fields.items():
+            setattr(upstream_frame, k, v)
+
+        downstream_frame.broadcast_sibling_id = upstream_frame.id
+        upstream_frame.broadcast_sibling_id = downstream_frame.id
+        await self.push_frame(downstream_frame)
+        await self.push_frame(upstream_frame, FrameDirection.UPSTREAM)
 
     async def __start(self, frame: StartFrame):
         """Handle the start frame to initialize processor state.
@@ -834,15 +892,7 @@ class FrameProcessor(BaseObject):
     async def _start_interruption(self):
         """Start handling an interruption by cancelling current tasks."""
         try:
-            if self._wait_for_interruption:
-                # If we get here we know the process task was just waiting for
-                # an interruption (push_interruption_task_frame_and_wait()), so
-                # we can't cancel the task because it might still need to do
-                # more things (e.g. pushing a frame after the
-                # interruption). Instead we just drain the queue because this is
-                # an interruption.
-                self.__reset_process_task()
-            elif isinstance(self.__process_current_frame, UninterruptibleFrame):
+            if isinstance(self.__process_current_frame, UninterruptibleFrame):
                 # We don't want to cancel UninterruptibleFrame, so we simply
                 # cleanup the queue.
                 self.__reset_process_queue()
@@ -866,7 +916,7 @@ class FrameProcessor(BaseObject):
         try:
             timestamp = self._clock.get_time() if self._clock else 0
             if direction == FrameDirection.DOWNSTREAM and self._next:
-                logger.trace(f"Pushing {frame} from {self} to {self._next}")
+                logger.trace(f"Pushing {frame} downstream from {self} to {self._next}")
 
                 if self._observer:
                     data = FramePushed(
@@ -950,7 +1000,8 @@ class FrameProcessor(BaseObject):
         # Process current queue and keep UninterruptibleFrame frames.
         while not self.__process_queue.empty():
             item = self.__process_queue.get_nowait()
-            if isinstance(item, UninterruptibleFrame):
+            frame = item[0]
+            if isinstance(frame, UninterruptibleFrame):
                 new_queue.put_nowait(item)
             self.__process_queue.task_done()
 

src/pipecat/processors/frameworks/rtvi/__init__.py

@@ -0,0 +1,73 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""RTVI (Real-Time Voice Interface) protocol implementation for Pipecat."""
+
+from pipecat.processors.frameworks.rtvi.frames import (
+    RTVIActionFrame,
+    RTVIClientMessageFrame,
+    RTVIServerMessageFrame,
+    RTVIServerResponseFrame,
+)
+from pipecat.processors.frameworks.rtvi.models_deprecated import (
+    ActionResult,
+    RTVIAction,
+    RTVIActionArgument,
+    RTVIActionArgumentData,
+    RTVIActionResponse,
+    RTVIActionResponseData,
+    RTVIActionRun,
+    RTVIActionRunArgument,
+    RTVIBotReadyDataDeprecated,
+    RTVIConfig,
+    RTVIConfigResponse,
+    RTVIDescribeActions,
+    RTVIDescribeActionsData,
+    RTVIDescribeConfig,
+    RTVIDescribeConfigData,
+    RTVIService,
+    RTVIServiceConfig,
+    RTVIServiceOption,
+    RTVIServiceOptionConfig,
+    RTVIUpdateConfig,
+)
+from pipecat.processors.frameworks.rtvi.observer import (
+    RTVIFunctionCallReportLevel,
+    RTVIObserver,
+    RTVIObserverParams,
+)
+from pipecat.processors.frameworks.rtvi.processor import RTVIProcessor
+
+__all__ = [
+    "ActionResult",
+    "RTVIAction",
+    "RTVIActionArgument",
+    "RTVIActionArgumentData",
+    "RTVIActionFrame",
+    "RTVIActionResponse",
+    "RTVIActionResponseData",
+    "RTVIActionRun",
+    "RTVIActionRunArgument",
+    "RTVIBotReadyDataDeprecated",
+    "RTVIClientMessageFrame",
+    "RTVIConfig",
+    "RTVIConfigResponse",
+    "RTVIDescribeActions",
+    "RTVIDescribeActionsData",
+    "RTVIDescribeConfig",
+    "RTVIDescribeConfigData",
+    "RTVIFunctionCallReportLevel",
+    "RTVIObserver",
+    "RTVIObserverParams",
+    "RTVIProcessor",
+    "RTVIServerMessageFrame",
+    "RTVIServerResponseFrame",
+    "RTVIService",
+    "RTVIServiceConfig",
+    "RTVIServiceOption",
+    "RTVIServiceOptionConfig",
+    "RTVIUpdateConfig",
+]

src/pipecat/processors/frameworks/rtvi/frames.py

@@ -0,0 +1,74 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""RTVI pipeline frame definitions."""
+
+from dataclasses import dataclass
+from typing import Any, Optional
+
+from pipecat.frames.frames import DataFrame, SystemFrame
+
+
+@dataclass
+class RTVIActionFrame(DataFrame):
+    """Frame containing an RTVI action to execute.
+
+    Parameters:
+        rtvi_action_run: The action to execute.
+        message_id: Optional message ID for response correlation.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    rtvi_action_run: Any
+    message_id: Optional[str] = None
+
+
+@dataclass
+class RTVIServerMessageFrame(SystemFrame):
+    """A frame for sending server messages to the client.
+
+    Parameters:
+        data: The message data to send to the client.
+    """
+
+    data: Any
+
+    def __str__(self):
+        """String representation of the RTVI server message frame."""
+        return f"{self.name}(data: {self.data})"
+
+
+@dataclass
+class RTVIClientMessageFrame(SystemFrame):
+    """A frame for sending messages from the client to the RTVI server.
+
+    This frame is meant for custom messaging from the client to the server
+    and expects a server-response message.
+    """
+
+    msg_id: str
+    type: str
+    data: Optional[Any] = None
+
+
+@dataclass
+class RTVIServerResponseFrame(SystemFrame):
+    """A frame for responding to a client RTVI message.
+
+    This frame should be sent in response to an RTVIClientMessageFrame
+    and include the original RTVIClientMessageFrame to ensure the response
+    is properly attributed to the original request. To respond with an error,
+    set the `error` field to a string describing the error. This will result
+    in the client receiving an `error-response` message instead of a
+    `server-response` message.
+    """
+
+    client_msg: RTVIClientMessageFrame
+    data: Optional[Any] = None
+    error: Optional[str] = None

src/pipecat/processors/frameworks/rtvi/models.py

@@ -0,0 +1,581 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""RTVI protocol v1 message models.
+
+Contains all RTVI protocol v1 message definitions and data structures.
+Import this module under the ``RTVI`` alias to use as a namespace::
+
+    import pipecat.processors.frameworks.rtvi.models as RTVI
+
+    msg = RTVI.BotReady(id="1", data=RTVI.BotReadyData(version=RTVI.PROTOCOL_VERSION))
+"""
+
+from typing import (
+    Any,
+    Dict,
+    Literal,
+    Mapping,
+    Optional,
+)
+
+from pydantic import BaseModel
+
+from pipecat.frames.frames import (
+    AggregationType,
+)
+
+# -- Constants --
+PROTOCOL_VERSION = "1.2.0"
+
+MESSAGE_LABEL = "rtvi-ai"
+MessageLiteral = Literal["rtvi-ai"]
+
+# -- Base Message Structure --
+
+
+class Message(BaseModel):
+    """Base RTVI message structure.
+
+    Represents the standard format for RTVI protocol messages.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: str
+    id: str
+    data: Optional[Dict[str, Any]] = None
+
+
+# -- Client -> Pipecat messages.
+
+
+class RawClientMessageData(BaseModel):
+    """Data structure expected from client messages sent to the RTVI server."""
+
+    t: str
+    d: Optional[Any] = None
+
+
+class ClientMessage(BaseModel):
+    """Cleansed data structure for client messages for handling."""
+
+    msg_id: str
+    type: str
+    data: Optional[Any] = None
+
+
+class RawServerResponseData(BaseModel):
+    """Data structure for server responses to client messages."""
+
+    t: str
+    d: Optional[Any] = None
+
+
+class ServerResponse(BaseModel):
+    """The RTVI-formatted message response from the server to the client.
+
+    This message is used to respond to custom messages sent by the client.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["server-response"] = "server-response"
+    id: str
+    data: RawServerResponseData
+
+
+class AboutClientData(BaseModel):
+    """Data about the RTVI client.
+
+    Contains information about the client, including which RTVI library it
+    is using, what platform it is on and any additional details, if available.
+    """
+
+    library: str
+    library_version: Optional[str] = None
+    platform: Optional[str] = None
+    platform_version: Optional[str] = None
+    platform_details: Optional[Any] = None
+
+
+class ClientReadyData(BaseModel):
+    """Data format of client ready messages.
+
+    Contains the RTVI protocol version and client information.
+    """
+
+    version: str
+    about: AboutClientData
+
+
+# -- Pipecat -> Client errors
+
+
+class ErrorResponseData(BaseModel):
+    """Data for an RTVI error response.
+
+    Contains the error message to send back to the client.
+    """
+
+    error: str
+
+
+class ErrorResponse(BaseModel):
+    """RTVI error response message.
+
+    RTVI formatted error response message for relaying failed client requests.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["error-response"] = "error-response"
+    id: str
+    data: ErrorResponseData
+
+
+class ErrorData(BaseModel):
+    """Data for an RTVI error event.
+
+    Contains error information including whether it's fatal.
+    """
+
+    error: str
+    fatal: bool  # Indicates the pipeline has stopped due to this error
+
+
+class Error(BaseModel):
+    """RTVI error event message.
+
+    RTVI formatted error message for relaying errors in the pipeline.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["error"] = "error"
+    data: ErrorData
+
+
+# -- Pipecat -> Client responses and messages.
+
+
+class BotReadyData(BaseModel):
+    """Data for bot ready notification.
+
+    Contains protocol version and initial configuration.
+    """
+
+    version: str
+    about: Optional[Mapping[str, Any]] = None
+
+
+class BotReady(BaseModel):
+    """Message indicating bot is ready for interaction.
+
+    Sent after bot initialization is complete.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-ready"] = "bot-ready"
+    id: str
+    data: BotReadyData
+
+
+class LLMFunctionCallMessageData(BaseModel):
+    """Data for LLM function call notification.
+
+    Contains function call details including name, ID, and arguments.
+
+    .. deprecated:: 0.0.102
+        Use ``LLMFunctionCallInProgressMessageData`` instead.
+    """
+
+    function_name: str
+    tool_call_id: str
+    args: Mapping[str, Any]
+
+
+class LLMFunctionCallMessage(BaseModel):
+    """Message notifying of an LLM function call.
+
+    Sent when the LLM makes a function call.
+
+    .. deprecated:: 0.0.102
+        Use ``LLMFunctionCallInProgressMessage`` with the
+        ``llm-function-call-in-progress`` event type instead.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["llm-function-call"] = "llm-function-call"
+    data: LLMFunctionCallMessageData
+
+
+class SendTextOptions(BaseModel):
+    """Options for sending text input to the LLM.
+
+    Contains options for how the pipeline should process the text input.
+    """
+
+    run_immediately: bool = True
+    audio_response: bool = True
+
+
+class SendTextData(BaseModel):
+    """Data format for sending text input to the LLM.
+
+    Contains the text content to send and any options for how the pipeline should process it.
+    """
+
+    content: str
+    options: Optional[SendTextOptions] = None
+
+
+class AppendToContextData(BaseModel):
+    """Data format for appending messages to the context.
+
+    Contains the role, content, and whether to run the message immediately.
+
+    .. deprecated:: 0.0.85
+        The RTVI message, append-to-context, has been deprecated. Use send-text
+        or custom client and server messages instead.
+    """
+
+    role: Literal["user", "assistant"] | str
+    content: Any
+    run_immediately: bool = False
+
+
+class AppendToContext(BaseModel):
+    """RTVI message format to append content to the LLM context.
+
+    .. deprecated:: 0.0.85
+        The RTVI message, append-to-context, has been deprecated. Use send-text
+        or custom client and server messages instead.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["append-to-context"] = "append-to-context"
+    data: AppendToContextData
+
+
+class LLMFunctionCallStartMessageData(BaseModel):
+    """Data for LLM function call start notification.
+
+    Contains the function name being called. Fields may be omitted based on
+    the configured function_call_report_level for security.
+    """
+
+    function_name: Optional[str] = None
+
+
+class LLMFunctionCallStartMessage(BaseModel):
+    """Message notifying that an LLM function call has started.
+
+    Sent when the LLM begins a function call.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["llm-function-call-started"] = "llm-function-call-started"
+    data: LLMFunctionCallStartMessageData
+
+
+class LLMFunctionCallResultData(BaseModel):
+    """Data for LLM function call result.
+
+    Contains function call details and result.
+    """
+
+    function_name: str
+    tool_call_id: str
+    arguments: dict
+    result: dict | str
+
+
+class LLMFunctionCallInProgressMessageData(BaseModel):
+    """Data for LLM function call in-progress notification.
+
+    Contains function call details including name, ID, and arguments.
+    Fields may be omitted based on the configured function_call_report_level for security.
+    """
+
+    tool_call_id: str
+    function_name: Optional[str] = None
+    arguments: Optional[Mapping[str, Any]] = None
+
+
+class LLMFunctionCallInProgressMessage(BaseModel):
+    """Message notifying that an LLM function call is in progress.
+
+    Sent when the LLM function call execution begins.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["llm-function-call-in-progress"] = "llm-function-call-in-progress"
+    data: LLMFunctionCallInProgressMessageData
+
+
+class LLMFunctionCallStoppedMessageData(BaseModel):
+    """Data for LLM function call stopped notification.
+
+    Contains details about the function call that stopped, including
+    whether it was cancelled or completed with a result.
+    Fields may be omitted based on the configured function_call_report_level for security.
+    """
+
+    tool_call_id: str
+    cancelled: bool
+    function_name: Optional[str] = None
+    result: Optional[Any] = None
+
+
+class LLMFunctionCallStoppedMessage(BaseModel):
+    """Message notifying that an LLM function call has stopped.
+
+    Sent when a function call completes (with result) or is cancelled.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["llm-function-call-stopped"] = "llm-function-call-stopped"
+    data: LLMFunctionCallStoppedMessageData
+
+
+class BotLLMStartedMessage(BaseModel):
+    """Message indicating bot LLM processing has started."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-llm-started"] = "bot-llm-started"
+
+
+class BotLLMStoppedMessage(BaseModel):
+    """Message indicating bot LLM processing has stopped."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-llm-stopped"] = "bot-llm-stopped"
+
+
+class BotTTSStartedMessage(BaseModel):
+    """Message indicating bot TTS processing has started."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-tts-started"] = "bot-tts-started"
+
+
+class BotTTSStoppedMessage(BaseModel):
+    """Message indicating bot TTS processing has stopped."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-tts-stopped"] = "bot-tts-stopped"
+
+
+class TextMessageData(BaseModel):
+    """Data for text-based RTVI messages.
+
+    Contains text content.
+    """
+
+    text: str
+
+
+class BotOutputMessageData(TextMessageData):
+    """Data for bot output RTVI messages.
+
+    Extends TextMessageData to include metadata about the output.
+    """
+
+    spoken: bool = False  # Indicates if the text has been spoken by TTS
+    aggregated_by: AggregationType | str
+    # Indicates what form the text is in (e.g., by word, sentence, etc.)
+
+
+class BotOutputMessage(BaseModel):
+    """Message containing bot output text.
+
+    An event meant to holistically represent what the bot is outputting,
+    along with metadata about the output and if it has been spoken.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-output"] = "bot-output"
+    data: BotOutputMessageData
+
+
+class BotTranscriptionMessage(BaseModel):
+    """Message containing bot transcription text.
+
+    Sent when the bot's speech is transcribed.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-transcription"] = "bot-transcription"
+    data: TextMessageData
+
+
+class BotLLMTextMessage(BaseModel):
+    """Message containing bot LLM text output.
+
+    Sent when the bot's LLM generates text.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-llm-text"] = "bot-llm-text"
+    data: TextMessageData
+
+
+class BotTTSTextMessage(BaseModel):
+    """Message containing bot TTS text output.
+
+    Sent when text is being processed by TTS.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-tts-text"] = "bot-tts-text"
+    data: TextMessageData
+
+
+class AudioMessageData(BaseModel):
+    """Data for audio-based RTVI messages.
+
+    Contains audio data and metadata.
+    """
+
+    audio: str
+    sample_rate: int
+    num_channels: int
+
+
+class BotTTSAudioMessage(BaseModel):
+    """Message containing bot TTS audio output.
+
+    Sent when the bot's TTS generates audio.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-tts-audio"] = "bot-tts-audio"
+    data: AudioMessageData
+
+
+class UserTranscriptionMessageData(BaseModel):
+    """Data for user transcription messages.
+
+    Contains transcription text and metadata.
+    """
+
+    text: str
+    user_id: str
+    timestamp: str
+    final: bool
+
+
+class UserTranscriptionMessage(BaseModel):
+    """Message containing user transcription.
+
+    Sent when user speech is transcribed.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["user-transcription"] = "user-transcription"
+    data: UserTranscriptionMessageData
+
+
+class UserLLMTextMessage(BaseModel):
+    """Message containing user text input for LLM.
+
+    Sent when user text is processed by the LLM.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["user-llm-text"] = "user-llm-text"
+    data: TextMessageData
+
+
+class UserStartedSpeakingMessage(BaseModel):
+    """Message indicating user has started speaking."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["user-started-speaking"] = "user-started-speaking"
+
+
+class UserStoppedSpeakingMessage(BaseModel):
+    """Message indicating user has stopped speaking."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["user-stopped-speaking"] = "user-stopped-speaking"
+
+
+class UserMuteStartedMessage(BaseModel):
+    """Message indicating user has been muted."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["user-mute-started"] = "user-mute-started"
+
+
+class UserMuteStoppedMessage(BaseModel):
+    """Message indicating user has been unmuted."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["user-mute-stopped"] = "user-mute-stopped"
+
+
+class BotStartedSpeakingMessage(BaseModel):
+    """Message indicating bot has started speaking."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-started-speaking"] = "bot-started-speaking"
+
+
+class BotStoppedSpeakingMessage(BaseModel):
+    """Message indicating bot has stopped speaking."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-stopped-speaking"] = "bot-stopped-speaking"
+
+
+class MetricsMessage(BaseModel):
+    """Message containing performance metrics.
+
+    Sent to provide performance and usage metrics.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["metrics"] = "metrics"
+    data: Mapping[str, Any]
+
+
+class ServerMessage(BaseModel):
+    """Generic server message.
+
+    Used for custom server-to-client messages.
+    """
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["server-message"] = "server-message"
+    data: Any
+
+
+class AudioLevelMessageData(BaseModel):
+    """Data format for sending audio levels."""
+
+    value: float
+
+
+class UserAudioLevelMessage(BaseModel):
+    """Message indicating user audio level."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["user-audio-level"] = "user-audio-level"
+    data: AudioLevelMessageData
+
+
+class BotAudioLevelMessage(BaseModel):
+    """Message indicating bot audio level."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["bot-audio-level"] = "bot-audio-level"
+    data: AudioLevelMessageData
+
+
+class SystemLogMessage(BaseModel):
+    """Message including a system log."""
+
+    label: MessageLiteral = MESSAGE_LABEL
+    type: Literal["system-log"] = "system-log"
+    data: TextMessageData

src/pipecat/processors/frameworks/rtvi/models_deprecated.py

@@ -0,0 +1,330 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""RTVI pre-1.0 protocol models (deprecated).
+
+All classes here are kept for backward compatibility only. Pipeline configuration
+and the actions API were removed in RTVI protocol 1.0.0. Use custom client and
+server messages instead.
+"""
+
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Optional,
+    Union,
+)
+
+from pydantic import BaseModel, Field, PrivateAttr
+
+import pipecat.processors.frameworks.rtvi.models as RTVI
+
+ActionResult = Union[bool, int, float, str, list, dict]
+
+
+class RTVIServiceOption(BaseModel):
+    """Configuration option for an RTVI service.
+
+    Defines a configurable option that can be set for an RTVI service,
+    including its name, type, and handler function.
+
+    .. deprecated:: 0.0.75
+        Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    name: str
+    type: Literal["bool", "number", "string", "array", "object"]
+    handler: Callable[..., Awaitable[None]] = Field(exclude=True)
+
+
+class RTVIService(BaseModel):
+    """An RTVI service definition.
+
+    Represents a service that can be configured and used within the RTVI protocol,
+    containing a name and list of configurable options.
+
+    .. deprecated:: 0.0.75
+       Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+       Use custom client and server messages instead.
+    """
+
+    name: str
+    options: List[RTVIServiceOption]
+    _options_dict: Dict[str, RTVIServiceOption] = PrivateAttr(default={})
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize the options dictionary after model creation."""
+        self._options_dict = {}
+        for option in self.options:
+            self._options_dict[option.name] = option
+        return super().model_post_init(__context)
+
+
+class RTVIActionArgumentData(BaseModel):
+    """Data for an RTVI action argument.
+
+    Contains the name and value of an argument passed to an RTVI action.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    name: str
+    value: Any
+
+
+class RTVIActionArgument(BaseModel):
+    """Definition of an RTVI action argument.
+
+    Specifies the name and expected type of an argument for an RTVI action.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    name: str
+    type: Literal["bool", "number", "string", "array", "object"]
+
+
+class RTVIAction(BaseModel):
+    """An RTVI action definition.
+
+    Represents an action that can be executed within the RTVI protocol,
+    including its service, name, arguments, and handler function.
+
+    .. deprecated:: 0.0.75
+       Actions have been removed as part of the RTVI protocol 1.0.0.
+       Use custom client and server messages instead.
+    """
+
+    service: str
+    action: str
+    arguments: List[RTVIActionArgument] = Field(default_factory=list)
+    result: Literal["bool", "number", "string", "array", "object"]
+    handler: Callable[..., Awaitable[ActionResult]] = Field(exclude=True)
+    _arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={})
+
+    def model_post_init(self, __context: Any) -> None:
+        """Initialize the arguments dictionary after model creation."""
+        self._arguments_dict = {}
+        for arg in self.arguments:
+            self._arguments_dict[arg.name] = arg
+        return super().model_post_init(__context)
+
+
+class RTVIServiceOptionConfig(BaseModel):
+    """Configuration value for an RTVI service option.
+
+    Contains the name and value to set for a specific service option.
+
+    .. deprecated:: 0.0.75
+        Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    name: str
+    value: Any
+
+
+class RTVIServiceConfig(BaseModel):
+    """Configuration for an RTVI service.
+
+    Contains the service name and list of option configurations to apply.
+
+    .. deprecated:: 0.0.75
+       Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+       Use custom client and server messages instead.
+    """
+
+    service: str
+    options: List[RTVIServiceOptionConfig]
+
+
+class RTVIConfig(BaseModel):
+    """Complete RTVI configuration.
+
+    Contains the full configuration for all RTVI services.
+
+    .. deprecated:: 0.0.75
+       Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+       Use custom client and server messages instead.
+    """
+
+    config: List[RTVIServiceConfig]
+
+
+#
+# Client -> Pipecat messages.
+#
+
+
+class RTVIUpdateConfig(BaseModel):
+    """Request to update RTVI configuration.
+
+    Contains new configuration settings and whether to interrupt the bot.
+
+    .. deprecated:: 0.0.75
+        Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    config: List[RTVIServiceConfig]
+    interrupt: bool = False
+
+
+class RTVIActionRunArgument(BaseModel):
+    """Argument for running an RTVI action.
+
+    Contains the name and value of an argument to pass to an action.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    name: str
+    value: Any
+
+
+class RTVIActionRun(BaseModel):
+    """Request to run an RTVI action.
+
+    Contains the service, action name, and optional arguments.
+
+    .. deprecated:: 0.0.75
+       Actions have been removed as part of the RTVI protocol 1.0.0.
+       Use custom client and server messages instead.
+    """
+
+    service: str
+    action: str
+    arguments: Optional[List[RTVIActionRunArgument]] = None
+
+
+#
+# Pipecat -> Client responses and messages.
+#
+
+
+class RTVIBotReadyDataDeprecated(RTVI.BotReadyData):
+    """Data for bot ready notification.
+
+    Contains protocol version and initial configuration.
+    """
+
+    # The config field is deprecated and will not be included if
+    # the client's rtvi version is 1.0.0 or higher.
+    config: Optional[List[RTVIServiceConfig]] = None
+
+
+class RTVIDescribeConfigData(BaseModel):
+    """Data for describing available RTVI configuration.
+
+    Contains the list of available services and their options.
+
+    .. deprecated:: 0.0.75
+        Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    config: List[RTVIService]
+
+
+class RTVIDescribeConfig(BaseModel):
+    """Message describing available RTVI configuration.
+
+    Sent in response to a describe-config request.
+
+    .. deprecated:: 0.0.75
+        Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    label: RTVI.MessageLiteral = RTVI.MESSAGE_LABEL
+    type: Literal["config-available"] = "config-available"
+    id: str
+    data: RTVIDescribeConfigData
+
+
+class RTVIDescribeActionsData(BaseModel):
+    """Data for describing available RTVI actions.
+
+    Contains the list of available actions that can be executed.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    actions: List[RTVIAction]
+
+
+class RTVIDescribeActions(BaseModel):
+    """Message describing available RTVI actions.
+
+    Sent in response to a describe-actions request.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    label: RTVI.MessageLiteral = RTVI.MESSAGE_LABEL
+    type: Literal["actions-available"] = "actions-available"
+    id: str
+    data: RTVIDescribeActionsData
+
+
+class RTVIConfigResponse(BaseModel):
+    """Response containing current RTVI configuration.
+
+    Sent in response to a get-config request.
+
+    .. deprecated:: 0.0.75
+        Pipeline Configuration has been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    label: RTVI.MessageLiteral = RTVI.MESSAGE_LABEL
+    type: Literal["config"] = "config"
+    id: str
+    data: RTVIConfig
+
+
+class RTVIActionResponseData(BaseModel):
+    """Data for an RTVI action response.
+
+    Contains the result of executing an action.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    result: ActionResult
+
+
+class RTVIActionResponse(BaseModel):
+    """Response to an RTVI action execution.
+
+    Sent after successfully executing an action.
+
+    .. deprecated:: 0.0.75
+        Actions have been removed as part of the RTVI protocol 1.0.0.
+        Use custom client and server messages instead.
+    """
+
+    label: RTVI.MessageLiteral = RTVI.MESSAGE_LABEL
+    type: Literal["action-response"] = "action-response"
+    id: str
+    data: RTVIActionResponseData

src/pipecat/processors/frameworks/rtvi/observer.py

@@ -0,0 +1,664 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""RTVI observer for converting pipeline frames to outgoing RTVI messages."""
+
+import time
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import (
+    TYPE_CHECKING,
+    Awaitable,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Set,
+    Tuple,
+)
+
+from loguru import logger
+from pydantic import BaseModel
+
+import pipecat.processors.frameworks.rtvi.models as RTVI
+from pipecat.audio.utils import calculate_audio_volume
+from pipecat.frames.frames import (
+    AggregatedTextFrame,
+    AggregationType,
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    Frame,
+    FunctionCallCancelFrame,
+    FunctionCallInProgressFrame,
+    FunctionCallResultFrame,
+    FunctionCallsStartedFrame,
+    InputAudioRawFrame,
+    InterimTranscriptionFrame,
+    LLMContextFrame,
+    LLMFullResponseEndFrame,
+    LLMFullResponseStartFrame,
+    LLMTextFrame,
+    MetricsFrame,
+    TranscriptionFrame,
+    TTSAudioRawFrame,
+    TTSStartedFrame,
+    TTSStoppedFrame,
+    TTSTextFrame,
+    UserMuteStartedFrame,
+    UserMuteStoppedFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+)
+from pipecat.metrics.metrics import (
+    LLMUsageMetricsData,
+    ProcessingMetricsData,
+    TTFBMetricsData,
+    TTSUsageMetricsData,
+)
+from pipecat.observers.base_observer import BaseObserver, FramePushed
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi.frames import (
+    RTVIServerMessageFrame,
+    RTVIServerResponseFrame,
+)
+from pipecat.transports.base_output import BaseOutputTransport
+from pipecat.utils.string import match_endofsentence
+
+if TYPE_CHECKING:
+    from pipecat.processors.frameworks.rtvi.processor import RTVIProcessor
+
+
+class RTVIFunctionCallReportLevel(str, Enum):
+    """Level of detail to include in function call RTVI events.
+
+    Controls what information is exposed in function call events for security.
+
+    Values:
+        DISABLED: No events emitted for this function call.
+        NONE: Events only with tool_call_id, no function name or metadata (most secure).
+        NAME: Events with function name, no arguments or results.
+        FULL: Events with function name, arguments, and results.
+    """
+
+    DISABLED = "disabled"
+    NONE = "none"
+    NAME = "name"
+    FULL = "full"
+
+
+@dataclass
+class RTVIObserverParams:
+    """Parameters for configuring RTVI Observer behavior.
+
+    .. deprecated:: 0.0.87
+        Parameter `errors_enabled` is deprecated. Error messages are always enabled.
+
+    Parameters:
+        bot_output_enabled: Indicates if bot output messages should be sent.
+        bot_llm_enabled: Indicates if the bot's LLM messages should be sent.
+        bot_tts_enabled: Indicates if the bot's TTS messages should be sent.
+        bot_speaking_enabled: Indicates if the bot's started/stopped speaking messages should be sent.
+        bot_audio_level_enabled: Indicates if bot's audio level messages should be sent.
+        user_llm_enabled: Indicates if the user's LLM input messages should be sent.
+        user_speaking_enabled: Indicates if the user's started/stopped speaking messages should be sent.
+        user_transcription_enabled: Indicates if user's transcription messages should be sent.
+        user_audio_level_enabled: Indicates if user's audio level messages should be sent.
+        metrics_enabled: Indicates if metrics messages should be sent.
+        system_logs_enabled: Indicates if system logs should be sent.
+        errors_enabled: [Deprecated] Indicates if errors messages should be sent.
+        ignored_sources: List of frame processors whose frames should be silently ignored
+            by this observer. Useful for suppressing RTVI messages from secondary pipeline
+            branches (e.g. a silent evaluation LLM) that should not be visible to clients.
+            Sources can also be added and removed dynamically via ``add_ignored_source()``
+            and ``remove_ignored_source()``.
+        skip_aggregator_types: List of aggregation types to skip sending as tts/output messages.
+            Note: if using this to avoid sending secure information, be sure to also disable
+            bot_llm_enabled to avoid leaking through LLM messages.
+        bot_output_transforms: A list of callables to transform text before just before sending it
+            to TTS. Each callable takes the aggregated text and its type, and returns the
+            transformed text. To register, provide a list of tuples of
+            (aggregation_type | '*', transform_function).
+        audio_level_period_secs: How often audio levels should be sent if enabled.
+        function_call_report_level: Controls what information is exposed in function call
+            events for security. A dict mapping function names to levels, where ``"*"``
+            sets the default level for unlisted functions::
+
+                function_call_report_level={
+                    "*": RTVIFunctionCallReportLevel.NONE,  # Default: events with no metadata
+                    "get_weather": RTVIFunctionCallReportLevel.FULL,  # Expose everything
+                }
+
+            Levels:
+                - DISABLED: No events emitted for this function.
+                - NONE: Events with tool_call_id only (most secure when events needed).
+                - NAME: Adds function name to events.
+                - FULL: Adds function name, arguments, and results.
+
+            Defaults to ``{"*": RTVIFunctionCallReportLevel.NONE}``.
+    """
+
+    bot_output_enabled: bool = True
+    bot_llm_enabled: bool = True
+    bot_tts_enabled: bool = True
+    bot_speaking_enabled: bool = True
+    bot_audio_level_enabled: bool = False
+    user_llm_enabled: bool = True
+    user_speaking_enabled: bool = True
+    user_mute_enabled: bool = True
+    user_transcription_enabled: bool = True
+    user_audio_level_enabled: bool = False
+    metrics_enabled: bool = True
+    system_logs_enabled: bool = False
+    errors_enabled: Optional[bool] = None
+    ignored_sources: List[FrameProcessor] = field(default_factory=list)
+    skip_aggregator_types: Optional[List[AggregationType | str]] = None
+    bot_output_transforms: Optional[
+        List[
+            Tuple[
+                AggregationType | str,
+                Callable[[str, AggregationType | str], Awaitable[str]],
+            ]
+        ]
+    ] = None
+    audio_level_period_secs: float = 0.15
+    function_call_report_level: Dict[str, RTVIFunctionCallReportLevel] = field(
+        default_factory=lambda: {"*": RTVIFunctionCallReportLevel.NONE}
+    )
+
+
+class RTVIObserver(BaseObserver):
+    """Pipeline frame observer for RTVI server message handling.
+
+    This observer monitors pipeline frames and converts them into appropriate RTVI messages
+    for client communication. It handles various frame types including speech events,
+    transcriptions, LLM responses, and TTS events.
+
+    Note:
+        This observer only handles outgoing messages. Incoming RTVI client messages
+        are handled by the RTVIProcessor.
+    """
+
+    def __init__(
+        self,
+        rtvi: Optional["RTVIProcessor"] = None,
+        *,
+        params: Optional[RTVIObserverParams] = None,
+        **kwargs,
+    ):
+        """Initialize the RTVI observer.
+
+        Args:
+            rtvi: The RTVI processor to push frames to.
+            params: Settings to enable/disable specific messages.
+            **kwargs: Additional arguments passed to parent class.
+        """
+        super().__init__(**kwargs)
+        self._rtvi = rtvi
+        self._params = params or RTVIObserverParams()
+
+        self._ignored_sources: Set[FrameProcessor] = set(self._params.ignored_sources)
+        self._frames_seen = set()
+
+        self._bot_transcription = ""
+        self._last_user_audio_level = 0
+        self._last_bot_audio_level = 0
+
+        # Track bot speaking state for queuing aggregated text frames
+        self._bot_is_speaking = False
+        self._queued_aggregated_text_frames: List[AggregatedTextFrame] = []
+
+        if self._params.system_logs_enabled:
+            self._system_logger_id = logger.add(self._logger_sink)
+
+        if self._params.errors_enabled is not None:
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("always")
+                warnings.warn(
+                    "Parameter `errors_enabled` is deprecated. Error messages are always enabled.",
+                    DeprecationWarning,
+                )
+
+        self._aggregation_transforms: List[
+            Tuple[AggregationType | str, Callable[[str, AggregationType | str], Awaitable[str]]]
+        ] = self._params.bot_output_transforms or []
+
+    def add_bot_output_transformer(
+        self,
+        transform_function: Callable[[str, AggregationType | str], Awaitable[str]],
+        aggregation_type: AggregationType | str = "*",
+    ):
+        """Transform text for a specific aggregation type before sending as Bot Output or TTS.
+
+        Args:
+            transform_function: The function to apply for transformation. This function should take
+                the text and aggregation type as input and return the transformed text.
+                Ex.: async def my_transform(text: str, aggregation_type: str) -> str:
+            aggregation_type: The type of aggregation to transform. This value defaults to "*" to
+                handle all text before sending to the client.
+        """
+        self._aggregation_transforms.append((aggregation_type, transform_function))
+
+    def remove_bot_output_transformer(
+        self,
+        transform_function: Callable[[str, AggregationType | str], Awaitable[str]],
+        aggregation_type: AggregationType | str = "*",
+    ):
+        """Remove a text transformer for a specific aggregation type.
+
+        Args:
+            transform_function: The function to remove.
+            aggregation_type: The type of aggregation to remove the transformer for.
+        """
+        self._aggregation_transforms = [
+            (agg_type, func)
+            for agg_type, func in self._aggregation_transforms
+            if not (agg_type == aggregation_type and func == transform_function)
+        ]
+
+    def add_ignored_source(self, source: FrameProcessor):
+        """Ignore all frames pushed by the given processor.
+
+        Any frame whose source matches ``source`` will be silently skipped,
+        preventing RTVI messages from being emitted for activity in that
+        processor. Useful for suppressing events from secondary pipeline
+        branches (e.g. a silent evaluation LLM) that should not be visible
+        to clients.
+
+        Args:
+            source: The frame processor to ignore.
+        """
+        self._ignored_sources.add(source)
+
+    def remove_ignored_source(self, source: FrameProcessor):
+        """Stop ignoring frames pushed by the given processor.
+
+        Reverses a previous call to ``add_ignored_source()``. If ``source``
+        was not previously ignored this is a no-op.
+
+        Args:
+            source: The frame processor to stop ignoring.
+        """
+        self._ignored_sources.discard(source)
+
+    def _get_function_call_report_level(self, function_name: str) -> RTVIFunctionCallReportLevel:
+        """Get the report level for a specific function call.
+
+        Args:
+            function_name: The name of the function to get the report level for.
+
+        Returns:
+            The report level for the function. Looks up the function name first,
+            then falls back to "*" key, then NONE.
+        """
+        levels = self._params.function_call_report_level
+        if function_name in levels:
+            return levels[function_name]
+        return levels.get("*", RTVIFunctionCallReportLevel.NONE)
+
+    async def _logger_sink(self, message):
+        """Logger sink so we can send system logs to RTVI clients."""
+        message = RTVI.SystemLogMessage(data=RTVI.TextMessageData(text=message))
+        await self.send_rtvi_message(message)
+
+    async def cleanup(self):
+        """Cleanup RTVI observer resources."""
+        await super().cleanup()
+        if self._params.system_logs_enabled:
+            logger.remove(self._system_logger_id)
+
+    async def send_rtvi_message(self, model: BaseModel, exclude_none: bool = True):
+        """Send an RTVI message.
+
+        By default, we push a transport frame. But this function can be
+        overridden by subclass to send RTVI messages in different ways.
+
+        Args:
+            model: The message to send.
+            exclude_none: Whether to exclude None values from the model dump.
+
+        """
+        if self._rtvi:
+            await self._rtvi.push_transport_message(model, exclude_none)
+
+    async def on_push_frame(self, data: FramePushed):
+        """Process a frame being pushed through the pipeline.
+
+        Args:
+            data: Frame push event data containing source, frame, direction, and timestamp.
+        """
+        src = data.source
+        frame = data.frame
+        direction = data.direction
+
+        # Frames from explicitly ignored sources are always skipped.
+        if self._ignored_sources and src in self._ignored_sources:
+            return
+
+        # For broadcast frames (pushed in both directions), only process
+        # the downstream copy to avoid sending duplicate RTVI messages.
+        if frame.broadcast_sibling_id is not None and direction != FrameDirection.DOWNSTREAM:
+            return
+
+        # If we have already seen this frame, let's skip it.
+        if frame.id in self._frames_seen:
+            return
+
+        # This tells whether the frame is already processed. If false, we will try
+        # again the next time we see the frame.
+        mark_as_seen = True
+
+        if (
+            isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame))
+            and self._params.user_speaking_enabled
+        ):
+            await self._handle_interruptions(frame)
+        elif (
+            isinstance(frame, (UserMuteStartedFrame, UserMuteStoppedFrame))
+            and self._params.user_mute_enabled
+        ):
+            await self._handle_user_mute(frame)
+        elif (
+            isinstance(frame, (BotStartedSpeakingFrame, BotStoppedSpeakingFrame))
+            and self._params.bot_speaking_enabled
+        ):
+            await self._handle_bot_speaking(frame)
+        elif (
+            isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame))
+            and self._params.user_transcription_enabled
+        ):
+            await self._handle_user_transcriptions(frame)
+        elif (
+            isinstance(frame, (OpenAILLMContextFrame, LLMContextFrame))
+            and self._params.user_llm_enabled
+        ):
+            await self._handle_context(frame)
+        elif isinstance(frame, LLMFullResponseStartFrame) and self._params.bot_llm_enabled:
+            await self.send_rtvi_message(RTVI.BotLLMStartedMessage())
+        elif isinstance(frame, LLMFullResponseEndFrame) and self._params.bot_llm_enabled:
+            await self.send_rtvi_message(RTVI.BotLLMStoppedMessage())
+        elif isinstance(frame, LLMTextFrame) and self._params.bot_llm_enabled:
+            await self._handle_llm_text_frame(frame)
+        elif isinstance(frame, TTSStartedFrame) and self._params.bot_tts_enabled:
+            await self.send_rtvi_message(RTVI.BotTTSStartedMessage())
+        elif isinstance(frame, TTSStoppedFrame) and self._params.bot_tts_enabled:
+            await self.send_rtvi_message(RTVI.BotTTSStoppedMessage())
+        elif isinstance(frame, AggregatedTextFrame) and (
+            self._params.bot_output_enabled or self._params.bot_tts_enabled
+        ):
+            if isinstance(frame, TTSTextFrame) and not isinstance(src, BaseOutputTransport):
+                # This check is to make sure we handle the frame when it has gone
+                # through the transport and has correct timing.
+                mark_as_seen = False
+            else:
+                await self._handle_aggregated_llm_text(frame)
+        elif isinstance(frame, MetricsFrame) and self._params.metrics_enabled:
+            await self._handle_metrics(frame)
+        elif isinstance(frame, FunctionCallsStartedFrame):
+            for function_call in frame.function_calls:
+                report_level = self._get_function_call_report_level(function_call.function_name)
+                if report_level == RTVIFunctionCallReportLevel.DISABLED:
+                    continue
+                data = RTVI.LLMFunctionCallStartMessageData()
+                if report_level in (
+                    RTVIFunctionCallReportLevel.NAME,
+                    RTVIFunctionCallReportLevel.FULL,
+                ):
+                    data.function_name = function_call.function_name
+                message = RTVI.LLMFunctionCallStartMessage(data=data)
+                await self.send_rtvi_message(message)
+        elif isinstance(frame, FunctionCallInProgressFrame):
+            report_level = self._get_function_call_report_level(frame.function_name)
+            if report_level != RTVIFunctionCallReportLevel.DISABLED:
+                data = RTVI.LLMFunctionCallInProgressMessageData(tool_call_id=frame.tool_call_id)
+                if report_level in (
+                    RTVIFunctionCallReportLevel.NAME,
+                    RTVIFunctionCallReportLevel.FULL,
+                ):
+                    data.function_name = frame.function_name
+                if report_level == RTVIFunctionCallReportLevel.FULL:
+                    data.arguments = frame.arguments
+                message = RTVI.LLMFunctionCallInProgressMessage(data=data)
+                await self.send_rtvi_message(message)
+        elif isinstance(frame, FunctionCallCancelFrame):
+            report_level = self._get_function_call_report_level(frame.function_name)
+            if report_level != RTVIFunctionCallReportLevel.DISABLED:
+                data = RTVI.LLMFunctionCallStoppedMessageData(
+                    tool_call_id=frame.tool_call_id,
+                    cancelled=True,
+                )
+                if report_level in (
+                    RTVIFunctionCallReportLevel.NAME,
+                    RTVIFunctionCallReportLevel.FULL,
+                ):
+                    data.function_name = frame.function_name
+                message = RTVI.LLMFunctionCallStoppedMessage(data=data)
+                await self.send_rtvi_message(message)
+        elif isinstance(frame, FunctionCallResultFrame):
+            report_level = self._get_function_call_report_level(frame.function_name)
+            if report_level != RTVIFunctionCallReportLevel.DISABLED:
+                data = RTVI.LLMFunctionCallStoppedMessageData(
+                    tool_call_id=frame.tool_call_id,
+                    cancelled=False,
+                )
+                if report_level in (
+                    RTVIFunctionCallReportLevel.NAME,
+                    RTVIFunctionCallReportLevel.FULL,
+                ):
+                    data.function_name = frame.function_name
+                if report_level == RTVIFunctionCallReportLevel.FULL:
+                    data.result = frame.result if frame.result else None
+                message = RTVI.LLMFunctionCallStoppedMessage(data=data)
+                await self.send_rtvi_message(message)
+        elif isinstance(frame, RTVIServerMessageFrame):
+            message = RTVI.ServerMessage(data=frame.data)
+            await self.send_rtvi_message(message)
+        elif isinstance(frame, RTVIServerResponseFrame):
+            if frame.error is not None:
+                await self._send_error_response(frame)
+            else:
+                await self._send_server_response(frame)
+        elif isinstance(frame, InputAudioRawFrame) and self._params.user_audio_level_enabled:
+            curr_time = time.time()
+            diff_time = curr_time - self._last_user_audio_level
+            if diff_time > self._params.audio_level_period_secs:
+                level = calculate_audio_volume(frame.audio, frame.sample_rate)
+                message = RTVI.UserAudioLevelMessage(data=RTVI.AudioLevelMessageData(value=level))
+                await self.send_rtvi_message(message)
+                self._last_user_audio_level = curr_time
+        elif isinstance(frame, TTSAudioRawFrame) and self._params.bot_audio_level_enabled:
+            curr_time = time.time()
+            diff_time = curr_time - self._last_bot_audio_level
+            if diff_time > self._params.audio_level_period_secs:
+                level = calculate_audio_volume(frame.audio, frame.sample_rate)
+                message = RTVI.BotAudioLevelMessage(data=RTVI.AudioLevelMessageData(value=level))
+                await self.send_rtvi_message(message)
+                self._last_bot_audio_level = curr_time
+
+        if mark_as_seen:
+            self._frames_seen.add(frame.id)
+
+    async def _handle_interruptions(self, frame: Frame):
+        """Handle user speaking interruption frames."""
+        message = None
+        if isinstance(frame, UserStartedSpeakingFrame):
+            message = RTVI.UserStartedSpeakingMessage()
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            message = RTVI.UserStoppedSpeakingMessage()
+
+        if message:
+            await self.send_rtvi_message(message)
+
+    async def _handle_user_mute(self, frame: Frame):
+        """Handle user mute/unmute frames."""
+        message = None
+        if isinstance(frame, UserMuteStartedFrame):
+            message = RTVI.UserMuteStartedMessage()
+        elif isinstance(frame, UserMuteStoppedFrame):
+            message = RTVI.UserMuteStoppedMessage()
+
+        if message:
+            await self.send_rtvi_message(message)
+
+    async def _handle_bot_speaking(self, frame: Frame):
+        """Handle bot speaking event frames."""
+        if isinstance(frame, BotStartedSpeakingFrame):
+            message = RTVI.BotStartedSpeakingMessage()
+            await self.send_rtvi_message(message)
+            # Flush any queued aggregated text frames
+            for queued_frame in self._queued_aggregated_text_frames:
+                await self._send_aggregated_llm_text(queued_frame)
+            self._queued_aggregated_text_frames.clear()
+            self._bot_is_speaking = True
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            message = RTVI.BotStoppedSpeakingMessage()
+            await self.send_rtvi_message(message)
+            self._bot_is_speaking = False
+
+    async def _handle_aggregated_llm_text(self, frame: AggregatedTextFrame):
+        """Handle aggregated LLM text output frames."""
+        if self._bot_is_speaking:
+            # Bot has already started speaking, send directly
+            await self._send_aggregated_llm_text(frame)
+        else:
+            # Bot hasn't started speaking yet, queue the frame
+            self._queued_aggregated_text_frames.append(frame)
+
+    async def _send_aggregated_llm_text(self, frame: AggregatedTextFrame):
+        """Send aggregated LLM text messages."""
+        # Skip certain aggregator types if configured to do so.
+        if (
+            self._params.skip_aggregator_types
+            and frame.aggregated_by in self._params.skip_aggregator_types
+        ):
+            return
+
+        text = frame.text
+        agg_type = frame.aggregated_by
+        for aggregation_type, transform in self._aggregation_transforms:
+            if aggregation_type == agg_type or aggregation_type == "*":
+                text = await transform(text, agg_type)
+
+        isTTS = isinstance(frame, TTSTextFrame)
+        if self._params.bot_output_enabled:
+            message = RTVI.BotOutputMessage(
+                data=RTVI.BotOutputMessageData(text=text, spoken=isTTS, aggregated_by=agg_type)
+            )
+            await self.send_rtvi_message(message)
+
+        if isTTS and self._params.bot_tts_enabled:
+            tts_message = RTVI.BotTTSTextMessage(data=RTVI.TextMessageData(text=text))
+            await self.send_rtvi_message(tts_message)
+
+    async def _handle_llm_text_frame(self, frame: LLMTextFrame):
+        """Handle LLM text output frames."""
+        message = RTVI.BotLLMTextMessage(data=RTVI.TextMessageData(text=frame.text))
+        await self.send_rtvi_message(message)
+
+        # TODO (mrkb): Remove all this logic when we fully deprecate bot-transcription messages.
+        self._bot_transcription += frame.text
+
+        if match_endofsentence(self._bot_transcription) and len(self._bot_transcription) > 0:
+            await self.send_rtvi_message(
+                RTVI.BotTranscriptionMessage(
+                    data=RTVI.TextMessageData(text=self._bot_transcription)
+                )
+            )
+            self._bot_transcription = ""
+
+    async def _handle_user_transcriptions(self, frame: Frame):
+        """Handle user transcription frames."""
+        message = None
+        if isinstance(frame, TranscriptionFrame):
+            message = RTVI.UserTranscriptionMessage(
+                data=RTVI.UserTranscriptionMessageData(
+                    text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=True
+                )
+            )
+        elif isinstance(frame, InterimTranscriptionFrame):
+            message = RTVI.UserTranscriptionMessage(
+                data=RTVI.UserTranscriptionMessageData(
+                    text=frame.text, user_id=frame.user_id, timestamp=frame.timestamp, final=False
+                )
+            )
+
+        if message:
+            await self.send_rtvi_message(message)
+
+    async def _handle_context(self, frame: OpenAILLMContextFrame | LLMContextFrame):
+        """Process LLM context frames to extract user messages for the RTVI client."""
+        try:
+            if isinstance(frame, OpenAILLMContextFrame):
+                messages = frame.context.messages
+            else:
+                messages = frame.context.get_messages()
+            if not messages:
+                return
+
+            message = messages[-1]
+
+            # Handle Google LLM format (protobuf objects with attributes)
+            # Note: not possible if frame is a universal LLMContextFrame
+            if hasattr(message, "role") and message.role == "user" and hasattr(message, "parts"):
+                text = "".join(part.text for part in message.parts if hasattr(part, "text"))
+                if text:
+                    rtvi_message = RTVI.UserLLMTextMessage(data=RTVI.TextMessageData(text=text))
+                    await self.send_rtvi_message(rtvi_message)
+
+            # Handle OpenAI format (original implementation)
+            elif isinstance(message, dict):
+                if message["role"] == "user":
+                    content = message["content"]
+                    if isinstance(content, list):
+                        text = " ".join(item["text"] for item in content if "text" in item)
+                    else:
+                        text = content
+                    rtvi_message = RTVI.UserLLMTextMessage(data=RTVI.TextMessageData(text=text))
+                    await self.send_rtvi_message(rtvi_message)
+
+        except Exception as e:
+            logger.warning(f"Caught an error while trying to handle context: {e}")
+
+    async def _handle_metrics(self, frame: MetricsFrame):
+        """Handle metrics frames and convert to RTVI metrics messages."""
+        metrics = {}
+        for d in frame.data:
+            if isinstance(d, TTFBMetricsData):
+                if "ttfb" not in metrics:
+                    metrics["ttfb"] = []
+                metrics["ttfb"].append(d.model_dump(exclude_none=True))
+            elif isinstance(d, ProcessingMetricsData):
+                if "processing" not in metrics:
+                    metrics["processing"] = []
+                metrics["processing"].append(d.model_dump(exclude_none=True))
+            elif isinstance(d, LLMUsageMetricsData):
+                if "tokens" not in metrics:
+                    metrics["tokens"] = []
+                metrics["tokens"].append(d.value.model_dump(exclude_none=True))
+            elif isinstance(d, TTSUsageMetricsData):
+                if "characters" not in metrics:
+                    metrics["characters"] = []
+                metrics["characters"].append(d.model_dump(exclude_none=True))
+
+        message = RTVI.MetricsMessage(data=metrics)
+        await self.send_rtvi_message(message)
+
+    async def _send_server_response(self, frame: RTVIServerResponseFrame):
+        """Send a response to the client for a specific request."""
+        message = RTVI.ServerResponse(
+            id=str(frame.client_msg.msg_id),
+            data=RTVI.RawServerResponseData(t=frame.client_msg.type, d=frame.data),
+        )
+        await self.send_rtvi_message(message)
+
+    async def _send_error_response(self, frame: RTVIServerResponseFrame):
+        """Send a response to the client for a specific request."""
+        message = RTVI.ErrorResponse(
+            id=str(frame.client_msg.msg_id), data=RTVI.ErrorResponseData(error=frame.error)
+        )
+        await self.send_rtvi_message(message)

src/pipecat/processors/frameworks/rtvi/processor.py

@@ -0,0 +1,649 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""RTVIProcessor: main RTVI protocol processor."""
+
+import asyncio
+import base64
+from typing import Any, Dict, Mapping, Optional
+
+from loguru import logger
+from pydantic import BaseModel, ValidationError
+
+import pipecat.processors.frameworks.rtvi.models as RTVI
+from pipecat import version as pipecat_version
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    EndTaskFrame,
+    ErrorFrame,
+    Frame,
+    FunctionCallResultFrame,
+    InputAudioRawFrame,
+    InputTransportMessageFrame,
+    LLMConfigureOutputFrame,
+    LLMMessagesAppendFrame,
+    OutputTransportMessageUrgentFrame,
+    StartFrame,
+    SystemFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi.frames import RTVIActionFrame, RTVIClientMessageFrame
+from pipecat.processors.frameworks.rtvi.models_deprecated import (
+    RTVIAction,
+    RTVIActionResponse,
+    RTVIActionResponseData,
+    RTVIActionRun,
+    RTVIBotReadyDataDeprecated,
+    RTVIConfig,
+    RTVIConfigResponse,
+    RTVIDescribeActions,
+    RTVIDescribeActionsData,
+    RTVIDescribeConfig,
+    RTVIDescribeConfigData,
+    RTVIService,
+    RTVIServiceConfig,
+    RTVIServiceOptionConfig,
+    RTVIUpdateConfig,
+)
+from pipecat.processors.frameworks.rtvi.observer import RTVIObserver, RTVIObserverParams
+from pipecat.services.llm_service import (
+    FunctionCallParams,  # TODO(aleix): we shouldn't import `services` from `processors`
+)
+from pipecat.transports.base_input import BaseInputTransport
+from pipecat.transports.base_transport import BaseTransport
+
+
+class RTVIProcessor(FrameProcessor):
+    """Main processor for handling RTVI protocol messages and actions.
+
+    This processor manages the RTVI protocol communication including client-server
+    handshaking, configuration management, action execution, and message routing.
+    It serves as the central hub for RTVI protocol operations.
+    """
+
+    def __init__(
+        self,
+        *,
+        config: Optional[RTVIConfig] = None,
+        transport: Optional[BaseTransport] = None,
+        **kwargs,
+    ):
+        """Initialize the RTVI processor.
+
+        Args:
+            config: Initial RTVI configuration.
+            transport: Transport layer for communication.
+            **kwargs: Additional arguments passed to parent class.
+        """
+        super().__init__(**kwargs)
+        self._config = config or RTVIConfig(config=[])
+
+        self._bot_ready = False
+        self._client_ready = False
+        self._client_ready_id = ""
+        # Default to 0.3.0 which is the last version before actually having a
+        # "client-version".
+        self._client_version = [0, 3, 0]
+        self._llm_skip_tts: bool = False  # Keep in sync with llm_service.py's configuration.
+
+        self._registered_actions: Dict[str, RTVIAction] = {}
+        self._registered_services: Dict[str, RTVIService] = {}
+
+        # A task to process incoming action frames.
+        self._action_task: Optional[asyncio.Task] = None
+
+        # A task to process incoming transport messages.
+        self._message_task: Optional[asyncio.Task] = None
+
+        self._register_event_handler("on_bot_started")
+        self._register_event_handler("on_client_ready")
+        self._register_event_handler("on_client_message")
+
+        self._input_transport = None
+        self._transport = transport
+        if self._transport:
+            input_transport = self._transport.input()
+            if isinstance(input_transport, BaseInputTransport):
+                self._input_transport = input_transport
+                self._input_transport.enable_audio_in_stream_on_start(False)
+
+    def register_action(self, action: RTVIAction):
+        """Register an action that can be executed via RTVI.
+
+        Args:
+            action: The action to register.
+        """
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "The actions API is deprecated, use server and client messages instead.",
+                DeprecationWarning,
+            )
+
+        id = self._action_id(action.service, action.action)
+        self._registered_actions[id] = action
+
+    def register_service(self, service: RTVIService):
+        """Register a service that can be configured via RTVI.
+
+        Args:
+            service: The service to register.
+        """
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "The actions API is deprecated, use server and client messages instead.",
+                DeprecationWarning,
+            )
+
+        self._registered_services[service.name] = service
+
+    def create_rtvi_observer(self, *, params: Optional[RTVIObserverParams] = None, **kwargs):
+        """Creates a new RTVI Observer.
+
+        Args:
+            params: Settings to enable/disable specific messages.
+            **kwargs: Additional arguments passed to the observer.
+
+        Returns:
+            A new RTVI observer.
+        """
+        return RTVIObserver(self, params=params, **kwargs)
+
+    async def set_client_ready(self):
+        """Mark the client as ready and trigger the ready event."""
+        self._client_ready = True
+        await self._call_event_handler("on_client_ready")
+
+    async def set_bot_ready(self, about: Mapping[str, Any] = None):
+        """Mark the bot as ready and send the bot-ready message.
+
+        Args:
+            about: Optional information about the bot to include in the ready message.
+                   If left as None, the Pipecat library and version will be used.
+        """
+        self._bot_ready = True
+        # Only call the (deprecated) _update_config method if the we're using a
+        # config (which is deprecated). Otherwise we'd always print an
+        # unnecessary deprecation warning.
+        if self._config.config:
+            await self._update_config(self._config, False)
+        await self._send_bot_ready(about=about)
+
+    async def interrupt_bot(self):
+        """Send a bot interruption frame upstream."""
+        await self.broadcast_interruption()
+
+    async def send_server_message(self, data: Any):
+        """Send a server message to the client."""
+        message = RTVI.ServerMessage(data=data)
+        await self._send_server_message(message)
+
+    async def send_server_response(self, client_msg: RTVI.ClientMessage, data: Any):
+        """Send a server response for a given client message."""
+        message = RTVI.ServerResponse(
+            id=client_msg.msg_id, data=RTVI.RawServerResponseData(t=client_msg.type, d=data)
+        )
+        await self._send_server_message(message)
+
+    async def send_error_response(self, client_msg: RTVI.ClientMessage, error: str):
+        """Send an error response for a given client message."""
+        await self._send_error_response(id=client_msg.msg_id, error=error)
+
+    async def send_error(self, error: str):
+        """Send an error message to the client.
+
+        Args:
+            error: The error message to send.
+        """
+        await self._send_error_frame(ErrorFrame(error=error))
+
+    async def push_transport_message(self, model: BaseModel, exclude_none: bool = True):
+        """Push a transport message frame."""
+        frame = OutputTransportMessageUrgentFrame(
+            message=model.model_dump(exclude_none=exclude_none)
+        )
+        await self.push_frame(frame)
+
+    async def handle_message(self, message: RTVI.Message):
+        """Handle an incoming RTVI message.
+
+        Args:
+            message: The RTVI message to handle.
+        """
+        await self._message_queue.put(message)
+
+    async def handle_function_call(self, params: FunctionCallParams):
+        """Handle a function call from the LLM.
+
+        Args:
+            params: The function call parameters.
+
+        .. deprecated:: 0.0.102
+            This method is deprecated. Function call events are now automatically
+            sent by ``RTVIObserver`` using the ``llm-function-call-in-progress`` event.
+            Configure reporting level via ``RTVIObserverParams.function_call_report_level``.
+        """
+        import warnings
+
+        warnings.warn(
+            "handle_function_call is deprecated. Function call events are now "
+            "automatically sent by RTVIObserver using llm-function-call-in-progress.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        fn = RTVI.LLMFunctionCallMessageData(
+            function_name=params.function_name,
+            tool_call_id=params.tool_call_id,
+            args=params.arguments,
+        )
+        message = RTVI.LLMFunctionCallMessage(data=fn)
+        await self.push_transport_message(message, exclude_none=False)
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames through the RTVI processor.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame flow.
+        """
+        await super().process_frame(frame, direction)
+
+        # Specific system frames
+        if isinstance(frame, StartFrame):
+            # Push StartFrame before start(), because we want StartFrame to be
+            # processed by every processor before any other frame is processed.
+            await self.push_frame(frame, direction)
+            await self._start(frame)
+        elif isinstance(frame, CancelFrame):
+            await self._cancel(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, ErrorFrame):
+            await self._send_error_frame(frame)
+            await self.push_frame(frame, direction)
+        elif isinstance(frame, InputTransportMessageFrame):
+            await self._handle_transport_message(frame)
+        # All other system frames
+        elif isinstance(frame, SystemFrame):
+            await self.push_frame(frame, direction)
+        # Control frames
+        elif isinstance(frame, EndFrame):
+            # Push EndFrame before stop(), because stop() waits on the task to
+            # finish and the task finishes when EndFrame is processed.
+            await self.push_frame(frame, direction)
+            await self._stop(frame)
+        # Data frames
+        elif isinstance(frame, RTVIActionFrame):
+            await self._action_queue.put(frame)
+        elif isinstance(frame, LLMConfigureOutputFrame):
+            self._llm_skip_tts = frame.skip_tts
+            await self.push_frame(frame, direction)
+        # Other frames
+        else:
+            await self.push_frame(frame, direction)
+
+    async def _start(self, frame: StartFrame):
+        """Start the RTVI processor tasks."""
+        if not self._action_task:
+            self._action_queue = asyncio.Queue()
+            self._action_task = self.create_task(self._action_task_handler())
+        if not self._message_task:
+            self._message_queue = asyncio.Queue()
+            self._message_task = self.create_task(self._message_task_handler())
+        await self._call_event_handler("on_bot_started")
+
+    async def _stop(self, frame: EndFrame):
+        """Stop the RTVI processor tasks."""
+        await self._cancel_tasks()
+
+    async def _cancel(self, frame: CancelFrame):
+        """Cancel the RTVI processor tasks."""
+        await self._cancel_tasks()
+
+    async def _cancel_tasks(self):
+        """Cancel all running tasks."""
+        if self._action_task:
+            await self.cancel_task(self._action_task)
+            self._action_task = None
+
+        if self._message_task:
+            await self.cancel_task(self._message_task)
+            self._message_task = None
+
+    async def _action_task_handler(self):
+        """Handle incoming action frames."""
+        while True:
+            frame = await self._action_queue.get()
+            await self._handle_action(frame.message_id, frame.rtvi_action_run)
+            self._action_queue.task_done()
+
+    async def _message_task_handler(self):
+        """Handle incoming transport messages."""
+        while True:
+            message = await self._message_queue.get()
+            await self._handle_message(message)
+            self._message_queue.task_done()
+
+    async def _handle_transport_message(self, frame: InputTransportMessageFrame):
+        """Handle an incoming transport message frame."""
+        try:
+            transport_message = frame.message
+            if transport_message.get("label") != RTVI.MESSAGE_LABEL:
+                logger.warning(f"Ignoring not RTVI message: {transport_message}")
+                return
+            message = RTVI.Message.model_validate(transport_message)
+            await self._message_queue.put(message)
+        except ValidationError as e:
+            await self.send_error(f"Invalid RTVI transport message: {e}")
+            logger.warning(f"Invalid RTVI transport message: {e}")
+
+    async def _handle_message(self, message: RTVI.Message):
+        """Handle a parsed RTVI message."""
+        try:
+            match message.type:
+                case "client-ready":
+                    data = None
+                    try:
+                        data = RTVI.ClientReadyData.model_validate(message.data)
+                    except ValidationError:
+                        # Not all clients have been updated to RTVI 1.0.0.
+                        # For now, that's okay, we just log their info as unknown.
+                        data = None
+                        pass
+                    await self._handle_client_ready(message.id, data)
+                case "describe-actions":
+                    await self._handle_describe_actions(message.id)
+                case "describe-config":
+                    await self._handle_describe_config(message.id)
+                case "get-config":
+                    await self._handle_get_config(message.id)
+                case "update-config":
+                    update_config = RTVIUpdateConfig.model_validate(message.data)
+                    await self._handle_update_config(message.id, update_config)
+                case "disconnect-bot":
+                    await self.push_frame(EndTaskFrame(), FrameDirection.UPSTREAM)
+                case "client-message":
+                    data = RTVI.RawClientMessageData.model_validate(message.data)
+                    await self._handle_client_message(message.id, data)
+                case "action":
+                    action = RTVIActionRun.model_validate(message.data)
+                    action_frame = RTVIActionFrame(message_id=message.id, rtvi_action_run=action)
+                    await self._action_queue.put(action_frame)
+                case "llm-function-call-result":
+                    data = RTVI.LLMFunctionCallResultData.model_validate(message.data)
+                    await self._handle_function_call_result(data)
+                case "send-text":
+                    data = RTVI.SendTextData.model_validate(message.data)
+                    await self._handle_send_text(data)
+                case "append-to-context":
+                    logger.warning(
+                        f"The append-to-context message is deprecated, use send-text instead."
+                    )
+                    data = RTVI.AppendToContextData.model_validate(message.data)
+                    await self._handle_update_context(data)
+                case "raw-audio" | "raw-audio-batch":
+                    await self._handle_audio_buffer(message.data)
+
+                case _:
+                    await self._send_error_response(message.id, f"Unsupported type {message.type}")
+
+        except ValidationError as e:
+            await self._send_error_response(message.id, f"Invalid message: {e}")
+            logger.warning(f"Invalid message: {e}")
+        except Exception as e:
+            await self._send_error_response(message.id, f"Exception processing message: {e}")
+            logger.warning(f"Exception processing message: {e}")
+
+    async def _handle_client_ready(self, request_id: str, data: RTVI.ClientReadyData | None):
+        """Handle the client-ready message from the client."""
+        version = data.version if data else None
+        logger.debug(f"Received client-ready: version {version}")
+        if version:
+            try:
+                self._client_version = [int(v) for v in version.split(".")]
+            except ValueError:
+                logger.warning(f"Invalid client version format: {version}")
+        about = data.about if data else {"library": "unknown"}
+        logger.debug(f"Client Details: {about}")
+        if self._input_transport:
+            await self._input_transport.start_audio_in_streaming()
+
+        self._client_ready_id = request_id
+        await self.set_client_ready()
+
+    async def _handle_audio_buffer(self, data):
+        """Handle incoming audio buffer data."""
+        if not self._input_transport:
+            return
+
+        # Extract audio batch ensuring it's a list
+        audio_list = data.get("base64AudioBatch") or [data.get("base64Audio")]
+
+        try:
+            for base64_audio in filter(None, audio_list):  # Filter out None values
+                pcm_bytes = base64.b64decode(base64_audio)
+                frame = InputAudioRawFrame(
+                    audio=pcm_bytes,
+                    sample_rate=data["sampleRate"],
+                    num_channels=data["numChannels"],
+                )
+                await self._input_transport.push_audio_frame(frame)
+
+        except (KeyError, TypeError, ValueError) as e:
+            # Handle missing keys, decoding errors, and invalid types
+            logger.error(f"Error processing audio buffer: {e}")
+
+    async def _handle_describe_config(self, request_id: str):
+        """Handle a describe-config request."""
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "Configuration helpers are deprecated. If your application needs this behavior, use custom server and client messages.",
+                DeprecationWarning,
+            )
+
+        services = list(self._registered_services.values())
+        message = RTVIDescribeConfig(id=request_id, data=RTVIDescribeConfigData(config=services))
+        await self.push_transport_message(message)
+
+    async def _handle_describe_actions(self, request_id: str):
+        """Handle a describe-actions request."""
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "The Actions API is deprecated, use custom server and client messages instead.",
+                DeprecationWarning,
+            )
+
+        actions = list(self._registered_actions.values())
+        message = RTVIDescribeActions(id=request_id, data=RTVIDescribeActionsData(actions=actions))
+        await self.push_transport_message(message)
+
+    async def _handle_get_config(self, request_id: str):
+        """Handle a get-config request."""
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "Configuration helpers are deprecated. If your application needs this behavior, use custom server and client messages.",
+                DeprecationWarning,
+            )
+
+        message = RTVIConfigResponse(id=request_id, data=self._config)
+        await self.push_transport_message(message)
+
+    def _update_config_option(self, service: str, config: RTVIServiceOptionConfig):
+        """Update a specific configuration option."""
+        for service_config in self._config.config:
+            if service_config.service == service:
+                for option_config in service_config.options:
+                    if option_config.name == config.name:
+                        option_config.value = config.value
+                        return
+                # If we couldn't find a value for this config, we simply need to
+                # add it.
+                service_config.options.append(config)
+
+    async def _update_service_config(self, config: RTVIServiceConfig):
+        """Update configuration for a specific service."""
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "Configuration helpers are deprecated. If your application needs this behavior, use custom server and client messages.",
+                DeprecationWarning,
+            )
+
+        service = self._registered_services[config.service]
+        for option in config.options:
+            handler = service._options_dict[option.name].handler
+            await handler(self, service.name, option)
+            self._update_config_option(service.name, option)
+
+    async def _update_config(self, data: RTVIConfig, interrupt: bool):
+        """Update the RTVI configuration."""
+        import warnings
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(
+                "Configuration helpers are deprecated. If your application needs this behavior, use custom server and client messages.",
+                DeprecationWarning,
+            )
+
+        if interrupt:
+            await self.interrupt_bot()
+        for service_config in data.config:
+            await self._update_service_config(service_config)
+
+    async def _handle_update_config(self, request_id: str, data: RTVIUpdateConfig):
+        """Handle an update-config request."""
+        await self._update_config(RTVIConfig(config=data.config), data.interrupt)
+        await self._handle_get_config(request_id)
+
+    async def _handle_send_text(self, data: RTVI.SendTextData):
+        """Handle a send-text message from the client."""
+        opts = data.options if data.options is not None else RTVI.SendTextOptions()
+        if opts.run_immediately:
+            await self.interrupt_bot()
+        cur_llm_skip_tts = self._llm_skip_tts
+        should_skip_tts = not opts.audio_response
+        toggle_skip_tts = cur_llm_skip_tts != should_skip_tts
+        if toggle_skip_tts:
+            output_frame = LLMConfigureOutputFrame(skip_tts=should_skip_tts)
+            await self.push_frame(output_frame)
+        text_frame = LLMMessagesAppendFrame(
+            messages=[{"role": "user", "content": data.content}],
+            run_llm=opts.run_immediately,
+        )
+        await self.push_frame(text_frame)
+        if toggle_skip_tts:
+            output_frame = LLMConfigureOutputFrame(skip_tts=cur_llm_skip_tts)
+            await self.push_frame(output_frame)
+
+    async def _handle_update_context(self, data: RTVI.AppendToContextData):
+        if data.run_immediately:
+            await self.interrupt_bot()
+        frame = LLMMessagesAppendFrame(
+            messages=[{"role": data.role, "content": data.content}],
+            run_llm=data.run_immediately,
+        )
+        await self.push_frame(frame)
+
+    async def _handle_client_message(self, msg_id: str, data: RTVI.RawClientMessageData):
+        """Handle a client message frame."""
+        # Create a RTVIClientMessageFrame to push the message
+        frame = RTVIClientMessageFrame(msg_id=msg_id, type=data.t, data=data.d)
+        await self.push_frame(frame)
+        await self._call_event_handler(
+            "on_client_message",
+            RTVI.ClientMessage(
+                msg_id=msg_id,
+                type=data.t,
+                data=data.d,
+            ),
+        )
+
+    async def _handle_function_call_result(self, data):
+        """Handle a function call result from the client."""
+        frame = FunctionCallResultFrame(
+            function_name=data.function_name,
+            tool_call_id=data.tool_call_id,
+            arguments=data.arguments,
+            result=data.result,
+        )
+        await self.push_frame(frame)
+
+    async def _handle_action(self, request_id: Optional[str], data: RTVIActionRun):
+        """Handle an action execution request."""
+        action_id = self._action_id(data.service, data.action)
+        if action_id not in self._registered_actions:
+            await self._send_error_response(request_id, f"Action {action_id} not registered")
+            return
+        action = self._registered_actions[action_id]
+        arguments = {}
+        if data.arguments:
+            for arg in data.arguments:
+                arguments[arg.name] = arg.value
+        result = await action.handler(self, action.service, arguments)
+        # Only send a response if request_id is present. Things that don't care about
+        # action responses (such as webhooks) don't set a request_id
+        if request_id:
+            message = RTVIActionResponse(id=request_id, data=RTVIActionResponseData(result=result))
+            await self.push_transport_message(message)
+
+    async def _send_bot_ready(self, about: Mapping[str, Any] = None):
+        """Send the bot-ready message to the client.
+
+        Args:
+            about: Optional information about the bot to include in the ready message.
+                   If left as None, the pipecat library and version will be used.
+        """
+        if not about:
+            about = {"library": "pipecat-ai", "library_version": f"{pipecat_version()}"}
+        if self._client_version and self._client_version[0] < 1:
+            config = self._config.config
+            message = RTVI.BotReady(
+                id=self._client_ready_id,
+                data=RTVIBotReadyDataDeprecated(
+                    version=RTVI.PROTOCOL_VERSION, about=about, config=config
+                ),
+            )
+        else:
+            message = RTVI.BotReady(
+                id=self._client_ready_id,
+                data=RTVI.BotReadyData(version=RTVI.PROTOCOL_VERSION, about=about),
+            )
+        await self.push_transport_message(message)
+
+    async def _send_server_message(self, message: RTVI.ServerMessage | RTVI.ServerResponse):
+        """Send a message or response to the client."""
+        await self.push_transport_message(message)
+
+    async def _send_error_frame(self, frame: ErrorFrame):
+        """Send an error frame as an RTVI error message."""
+        message = RTVI.Error(data=RTVI.ErrorData(error=frame.error, fatal=frame.fatal))
+        await self.push_transport_message(message)
+
+    async def _send_error_response(self, id: str, error: str):
+        """Send an error response message."""
+        message = RTVI.ErrorResponse(id=id, data=RTVI.ErrorResponseData(error=error))
+        await self.push_transport_message(message)
+
+    def _action_id(self, service: str, action: str) -> str:
+        """Generate an action ID from service and action names."""
+        return f"{service}:{action}"

src/pipecat/processors/metrics/frame_processor_metrics.py

@@ -17,6 +17,7 @@ from pipecat.metrics.metrics import (
     LLMUsageMetricsData,
     MetricsData,
     ProcessingMetricsData,
+    TextAggregationMetricsData,
     TTFBMetricsData,
     TTSUsageMetricsData,
 )
@@ -43,6 +44,7 @@ class FrameProcessorMetrics(BaseObject):
         self._task_manager = None
         self._start_ttfb_time = 0
         self._start_processing_time = 0
+        self._start_text_aggregation_time = 0
         self._last_ttfb_time = 0
         self._should_report_ttfb = True
 
@@ -107,49 +109,70 @@ class FrameProcessorMetrics(BaseObject):
         """
         self._core_metrics_data = MetricsData(processor=name)
 
-    async def start_ttfb_metrics(self, report_only_initial_ttfb):
+    async def start_ttfb_metrics(
+        self, *, start_time: Optional[float] = None, report_only_initial_ttfb: bool
+    ):
         """Start measuring time-to-first-byte (TTFB).
 
         Args:
+            start_time: Optional timestamp to use as the start time. If None,
+                uses the current time.
             report_only_initial_ttfb: Whether to report only the first TTFB measurement.
         """
         if self._should_report_ttfb:
-            self._start_ttfb_time = time.time()
+            self._start_ttfb_time = start_time or time.time()
             self._last_ttfb_time = 0
             self._should_report_ttfb = not report_only_initial_ttfb
 
-    async def stop_ttfb_metrics(self):
+    async def stop_ttfb_metrics(self, *, end_time: Optional[float] = None):
         """Stop TTFB measurement and generate metrics frame.
 
+        Args:
+            end_time: Optional timestamp to use as the end time. If None, uses
+                the current time.
+
         Returns:
             MetricsFrame containing TTFB data, or None if not measuring.
         """
         if self._start_ttfb_time == 0:
             return None
 
-        self._last_ttfb_time = time.time() - self._start_ttfb_time
-        logger.debug(f"{self._processor_name()} TTFB: {self._last_ttfb_time}")
+        end_time = end_time or time.time()
+
+        self._last_ttfb_time = end_time - self._start_ttfb_time
+        logger.debug(f"{self._processor_name()} TTFB: {self._last_ttfb_time:.3f}s")
         ttfb = TTFBMetricsData(
             processor=self._processor_name(), value=self._last_ttfb_time, model=self._model_name()
         )
         self._start_ttfb_time = 0
         return MetricsFrame(data=[ttfb])
 
-    async def start_processing_metrics(self):
-        """Start measuring processing time."""
-        self._start_processing_time = time.time()
+    async def start_processing_metrics(self, *, start_time: Optional[float] = None):
+        """Start measuring processing time.
 
-    async def stop_processing_metrics(self):
+        Args:
+            start_time: Optional timestamp to use as the start time. If None,
+                uses the current time.
+        """
+        self._start_processing_time = start_time or time.time()
+
+    async def stop_processing_metrics(self, *, end_time: Optional[float] = None):
         """Stop processing time measurement and generate metrics frame.
 
+        Args:
+            end_time: Optional timestamp to use as the end time. If None, uses
+                the current time.
+
         Returns:
             MetricsFrame containing processing duration data, or None if not measuring.
         """
         if self._start_processing_time == 0:
             return None
 
-        value = time.time() - self._start_processing_time
-        logger.debug(f"{self._processor_name()} processing time: {value}")
+        end_time = end_time or time.time()
+
+        value = end_time - self._start_processing_time
+        logger.debug(f"{self._processor_name()} processing time: {value:.3f}s")
         processing = ProcessingMetricsData(
             processor=self._processor_name(), value=value, model=self._model_name()
         )
@@ -190,3 +213,24 @@ class FrameProcessorMetrics(BaseObject):
         )
         logger.debug(f"{self._processor_name()} usage characters: {characters.value}")
         return MetricsFrame(data=[characters])
+
+    async def start_text_aggregation_metrics(self):
+        """Start measuring text aggregation time (first token to first sentence)."""
+        self._start_text_aggregation_time = time.time()
+
+    async def stop_text_aggregation_metrics(self):
+        """Stop text aggregation measurement and generate metrics frame.
+
+        Returns:
+            MetricsFrame containing text aggregation time, or None if not measuring.
+        """
+        if self._start_text_aggregation_time == 0:
+            return None
+
+        value = time.time() - self._start_text_aggregation_time
+        logger.debug(f"{self._processor_name()} text aggregation time: {value}")
+        aggregation = TextAggregationMetricsData(
+            processor=self._processor_name(), value=value, model=self._model_name()
+        )
+        self._start_text_aggregation_time = 0
+        return MetricsFrame(data=[aggregation])

src/pipecat/processors/metrics/sentry.py

@@ -7,6 +7,7 @@
 """Sentry integration for frame processor metrics."""
 
 import asyncio
+from typing import Optional
 
 from loguru import logger
 
@@ -70,13 +71,18 @@ class SentryMetrics(FrameProcessorMetrics):
             logger.trace(f"{self} Flushing Sentry metrics")
             sentry_sdk.flush(timeout=5.0)
 
-    async def start_ttfb_metrics(self, report_only_initial_ttfb):
+    async def start_ttfb_metrics(
+        self, *, start_time: Optional[float] = None, report_only_initial_ttfb: bool
+    ):
         """Start tracking time-to-first-byte metrics.
 
         Args:
+            start_time: Optional start timestamp override.
             report_only_initial_ttfb: Whether to report only the initial TTFB measurement.
         """
-        await super().start_ttfb_metrics(report_only_initial_ttfb)
+        await super().start_ttfb_metrics(
+            start_time=start_time, report_only_initial_ttfb=report_only_initial_ttfb
+        )
 
         if self._should_report_ttfb and self._sentry_available:
             self._ttfb_metrics_tx = sentry_sdk.start_transaction(
@@ -87,23 +93,25 @@ class SentryMetrics(FrameProcessorMetrics):
                 f"{self} Sentry transaction started (ID: {self._ttfb_metrics_tx.span_id} Name: {self._ttfb_metrics_tx.name})"
             )
 
-    async def stop_ttfb_metrics(self):
+    async def stop_ttfb_metrics(self, *, end_time: Optional[float] = None):
         """Stop tracking time-to-first-byte metrics.
 
-        Queues the TTFB transaction for completion and transmission to Sentry.
+        Args:
+            end_time: Optional end timestamp override.
         """
-        await super().stop_ttfb_metrics()
+        await super().stop_ttfb_metrics(end_time=end_time)
 
         if self._sentry_available and self._ttfb_metrics_tx:
             await self._sentry_queue.put(self._ttfb_metrics_tx)
             self._ttfb_metrics_tx = None
 
-    async def start_processing_metrics(self):
+    async def start_processing_metrics(self, *, start_time: Optional[float] = None):
         """Start tracking frame processing metrics.
 
-        Creates a new Sentry transaction to track processing performance.
+        Args:
+            start_time: Optional start timestamp override.
         """
-        await super().start_processing_metrics()
+        await super().start_processing_metrics(start_time=start_time)
 
         if self._sentry_available:
             self._processing_metrics_tx = sentry_sdk.start_transaction(
@@ -114,12 +122,13 @@ class SentryMetrics(FrameProcessorMetrics):
                 f"{self} Sentry transaction started (ID: {self._processing_metrics_tx.span_id} Name: {self._processing_metrics_tx.name})"
             )
 
-    async def stop_processing_metrics(self):
+    async def stop_processing_metrics(self, *, end_time: Optional[float] = None):
         """Stop tracking frame processing metrics.
 
-        Queues the processing transaction for completion and transmission to Sentry.
+        Args:
+            end_time: Optional end timestamp override.
         """
-        await super().stop_processing_metrics()
+        await super().stop_processing_metrics(end_time=end_time)
 
         if self._sentry_available and self._processing_metrics_tx:
             await self._sentry_queue.put(self._processing_metrics_tx)

src/pipecat/processors/user_idle_processor.py

@@ -8,6 +8,7 @@
 
 import asyncio
 import inspect
+import warnings
 from typing import Awaitable, Callable, Union
 
 from pipecat.frames.frames import (
@@ -26,6 +27,10 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 class UserIdleProcessor(FrameProcessor):
     """Monitors user inactivity and triggers callbacks after timeout periods.
 
+    .. deprecated:: 0.0.100
+        UserIdleProcessor is deprecated in 0.0.100 and will be removed in a future version.
+        Use LLMUserAggregator with user_idle_timeout parameter instead.
+
     This processor tracks user activity and triggers configurable callbacks when
     users become idle. It starts monitoring only after the first conversation
     activity and supports both basic and retry-based callback patterns.
@@ -70,6 +75,14 @@ class UserIdleProcessor(FrameProcessor):
             **kwargs: Additional arguments passed to FrameProcessor.
         """
         super().__init__(**kwargs)
+
+        warnings.warn(
+            "UserIdleProcessor is deprecated in 0.0.100 and will be removed in a "
+            "future version. Use LLMUserAggregator with user_idle_timeout parameter "
+            "instead.",
+            DeprecationWarning,
+        )
+
         self._callback = self._wrap_callback(callback)
         self._timeout = timeout
         self._retry_count = 0

src/pipecat/runner/daily.py

@@ -17,7 +17,7 @@ Functions:
 Environment variables:
 
 - DAILY_API_KEY - Daily API key for room/token creation (required)
-- DAILY_SAMPLE_ROOM_URL (optional) - Existing room URL to use. If not provided,
+- DAILY_ROOM_URL (optional) - Existing room URL to use. If not provided,
   a temporary room will be created automatically.
 
 Example::
@@ -79,19 +79,22 @@ async def configure(
     aiohttp_session: aiohttp.ClientSession,
     *,
     api_key: Optional[str] = None,
-    room_exp_duration: Optional[float] = 2.0,
-    token_exp_duration: Optional[float] = 2.0,
+    room_exp_duration: float = 2.0,
+    token_exp_duration: float = 2.0,
     sip_caller_phone: Optional[str] = None,
-    sip_enable_video: Optional[bool] = False,
-    sip_num_endpoints: Optional[int] = 1,
+    sip_enable_video: bool = False,
+    sip_num_endpoints: int = 1,
+    enable_dialout: bool = False,
     sip_codecs: Optional[Dict[str, List[str]]] = None,
+    sip_provider: Optional[str] = None,
+    room_geo: Optional[str] = None,
     room_properties: Optional[DailyRoomProperties] = None,
-    token_properties: Optional["DailyMeetingTokenProperties"] = None,
+    token_properties: Optional[DailyMeetingTokenProperties] = None,
 ) -> DailyRoomConfig:
     """Configure Daily room URL and token with optional SIP capabilities.
 
     This function will either:
-    1. Use an existing room URL from DAILY_SAMPLE_ROOM_URL environment variable (standard mode only)
+    1. Use an existing room URL from DAILY_ROOM_URL environment variable (standard mode only)
     2. Create a new temporary room automatically if no URL is provided
 
     Args:
@@ -103,8 +106,14 @@ async def configure(
             When provided, enables SIP functionality and returns SipRoomConfig.
         sip_enable_video: Whether video is enabled for SIP.
         sip_num_endpoints: Number of allowed SIP endpoints.
+        enable_dialout: Whether to enable outbound dialing (PSTN or SIP) on the room.
+            Requires dial-out entitlement on your Daily account.
         sip_codecs: Codecs to support for audio and video. If None, uses Daily defaults.
             Example: {"audio": ["OPUS"], "video": ["H264"]}
+        sip_provider: SIP provider name (e.g., "daily"). Only used when
+            sip_caller_phone is provided and room_properties is not.
+        room_geo: Daily room geographic region (e.g., "us-east-1"). Only used
+            when room_properties is not provided.
         room_properties: Optional DailyRoomProperties to use instead of building from
             individual parameters. When provided, this overrides room_exp_duration and
             SIP-related parameters. If not provided, properties are built from the
@@ -153,7 +162,10 @@ async def configure(
                 sip_caller_phone is not None,
                 sip_enable_video is not False,
                 sip_num_endpoints != 1,
+                enable_dialout is not False,
                 sip_codecs is not None,
+                sip_provider is not None,
+                room_geo is not None,
             ]
         )
         if individual_params_provided:
@@ -176,23 +188,26 @@ async def configure(
         aiohttp_session=aiohttp_session,
     )
 
+    token_expiry_seconds: float = token_exp_duration * 60 * 60
+
     # Check for existing room URL (only in standard mode)
-    existing_room_url = os.getenv("DAILY_SAMPLE_ROOM_URL")
+    existing_room_url = os.getenv("DAILY_ROOM_URL")
     if existing_room_url and not sip_enabled:
         # Use existing room (standard mode only)
         logger.info(f"Using existing Daily room: {existing_room_url}")
         room_url = existing_room_url
 
         # Create token and return standard format
-        expiry_time: float = token_exp_duration * 60 * 60
         token_params = None
         if token_properties:
             token_params = DailyMeetingTokenParams(properties=token_properties)
-        token = await daily_rest_helper.get_token(room_url, expiry_time, params=token_params)
+        token = await daily_rest_helper.get_token(
+            room_url, token_expiry_seconds, params=token_params
+        )
         return DailyRoomConfig(room_url=room_url, token=token)
 
     # Create a new room
-    room_prefix = "pipecat-sip" if sip_enabled else "pipecat"
+    room_prefix = "pipecat-telephony" if (sip_enabled or enable_dialout) else "pipecat"
     room_name = f"{room_prefix}-{uuid.uuid4().hex[:8]}"
     logger.info(f"Creating new Daily room: {room_name}")
 
@@ -207,6 +222,12 @@ async def configure(
             eject_at_room_exp=True,
         )
 
+        if room_geo:
+            room_properties.geo = room_geo
+
+        if enable_dialout:
+            room_properties.enable_dialout = True
+
         # Add SIP configuration if enabled
         if sip_enabled:
             sip_params = DailyRoomSipParams(
@@ -215,9 +236,9 @@ async def configure(
                 sip_mode="dial-in",
                 num_endpoints=sip_num_endpoints,
                 codecs=sip_codecs,
+                provider=sip_provider,
             )
             room_properties.sip = sip_params
-            room_properties.enable_dialout = True  # Enable outbound calls if needed
             room_properties.start_video_off = not sip_enable_video  # Voice-only by default
 
     # Create room parameters
@@ -229,7 +250,6 @@ async def configure(
         logger.info(f"Created Daily room: {room_url}")
 
         # Create meeting token
-        token_expiry_seconds = token_exp_duration * 60 * 60
         token_params = None
         if token_properties:
             token_params = DailyMeetingTokenParams(properties=token_properties)

src/pipecat/runner/run.py

@@ -153,26 +153,18 @@ def _get_bot_module():
     )
 
 
-async def _run_telephony_bot(websocket: WebSocket):
+async def _run_telephony_bot(websocket: WebSocket, args: argparse.Namespace):
     """Run a bot for telephony transports."""
     bot_module = _get_bot_module()
 
     # Just pass the WebSocket - let the bot handle parsing
     runner_args = WebSocketRunnerArguments(websocket=websocket)
+    runner_args.cli_args = args
 
     await bot_module.bot(runner_args)
 
 
-def _create_server_app(
-    *,
-    transport_type: str,
-    host: str = "localhost",
-    proxy: str,
-    esp32_mode: bool = False,
-    whatsapp_enabled: bool = False,
-    folder: Optional[str] = None,
-    dialin_enabled: bool = False,
-):
+def _create_server_app(args: argparse.Namespace):
     """Create FastAPI app with transport-specific routes."""
     app = FastAPI()
 
@@ -185,23 +177,21 @@ def _create_server_app(
     )
 
     # Set up transport-specific routes
-    if transport_type == "webrtc":
-        _setup_webrtc_routes(app, esp32_mode=esp32_mode, host=host, folder=folder)
-        if whatsapp_enabled:
-            _setup_whatsapp_routes(app)
-    elif transport_type == "daily":
-        _setup_daily_routes(app, dialin_enabled=dialin_enabled)
-    elif transport_type in TELEPHONY_TRANSPORTS:
-        _setup_telephony_routes(app, transport_type=transport_type, proxy=proxy)
+    if args.transport == "webrtc":
+        _setup_webrtc_routes(app, args)
+        if args.whatsapp:
+            _setup_whatsapp_routes(app, args)
+    elif args.transport == "daily":
+        _setup_daily_routes(app, args)
+    elif args.transport in TELEPHONY_TRANSPORTS:
+        _setup_telephony_routes(app, args)
     else:
-        logger.warning(f"Unknown transport type: {transport_type}")
+        logger.warning(f"Unknown transport type: {args.transport}")
 
     return app
 
 
-def _setup_webrtc_routes(
-    app: FastAPI, *, esp32_mode: bool = False, host: str = "localhost", folder: Optional[str] = None
-):
+def _setup_webrtc_routes(app: FastAPI, args: argparse.Namespace):
     """Set up WebRTC-specific routes."""
     try:
         from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
@@ -241,11 +231,11 @@ def _setup_webrtc_routes(
     @app.get("/files/{filename:path}")
     async def download_file(filename: str):
         """Handle file downloads."""
-        if not folder:
+        if not args.folder:
             logger.warning(f"Attempting to dowload {filename}, but downloads folder not setup.")
             return
 
-        file_path = Path(folder) / filename
+        file_path = Path(args.folder) / filename
         if not os.path.exists(file_path):
             raise HTTPException(404)
 
@@ -255,7 +245,7 @@ def _setup_webrtc_routes(
 
     # Initialize the SmallWebRTC request handler
     small_webrtc_handler: SmallWebRTCRequestHandler = SmallWebRTCRequestHandler(
-        esp32_mode=esp32_mode, host=host
+        esp32_mode=args.esp32, host=args.host
     )
 
     @app.post("/api/offer")
@@ -263,12 +253,13 @@ def _setup_webrtc_routes(
         """Handle WebRTC offer requests via SmallWebRTCRequestHandler."""
 
         # Prepare runner arguments with the callback to run your bot
-        async def webrtc_connection_callback(connection):
+        async def webrtc_connection_callback(connection: SmallWebRTCConnection):
             bot_module = _get_bot_module()
 
             runner_args = SmallWebRTCRunnerArguments(
                 webrtc_connection=connection, body=request.request_data
             )
+            runner_args.cli_args = args
             background_tasks.add_task(bot_module.bot, runner_args)
 
         # Delegate handling to SmallWebRTCRequestHandler
@@ -298,7 +289,7 @@ def _setup_webrtc_routes(
 
         # Store session info immediately in memory, replicate the behavior expected on Pipecat Cloud
         session_id = str(uuid.uuid4())
-        active_sessions[session_id] = request_data
+        active_sessions[session_id] = request_data.get("body", {})
 
         result: StartBotResult = {"sessionId": session_id}
         if request_data.get("enableDefaultIceServers"):
@@ -331,7 +322,8 @@ def _setup_webrtc_routes(
                         pc_id=request_data.get("pc_id"),
                         restart_pc=request_data.get("restart_pc"),
                         request_data=request_data.get("request_data")
-                        or request_data.get("requestData"),
+                        or request_data.get("requestData")
+                        or active_session,
                     )
                     return await offer(webrtc_request, background_tasks)
                 elif request.method == HTTPMethod.PATCH.value:
@@ -380,8 +372,8 @@ def _add_lifespan_to_app(app: FastAPI, new_lifespan):
         app.router.lifespan_context = new_lifespan
 
 
-def _setup_whatsapp_routes(app: FastAPI):
-    """Set up WebRTC-specific routes."""
+def _setup_whatsapp_routes(app: FastAPI, args: argparse.Namespace):
+    """Set up WhatsApp-specific routes."""
     WHATSAPP_APP_SECRET = os.getenv("WHATSAPP_APP_SECRET")
     WHATSAPP_PHONE_NUMBER_ID = os.getenv("WHATSAPP_PHONE_NUMBER_ID")
     WHATSAPP_TOKEN = os.getenv("WHATSAPP_TOKEN")
@@ -406,13 +398,7 @@ def _setup_whatsapp_routes(app: FastAPI):
         return
 
     try:
-        from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
-
         from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
-        from pipecat.transports.smallwebrtc.request_handler import (
-            SmallWebRTCRequest,
-            SmallWebRTCRequestHandler,
-        )
         from pipecat.transports.whatsapp.api import WhatsAppWebhookRequest
         from pipecat.transports.whatsapp.client import WhatsAppClient
     except ImportError as e:
@@ -489,6 +475,7 @@ def _setup_whatsapp_routes(app: FastAPI):
             """
             bot_module = _get_bot_module()
             runner_args = SmallWebRTCRunnerArguments(webrtc_connection=connection)
+            runner_args.cli_args = args
             background_tasks.add_task(bot_module.bot, runner_args)
 
         try:
@@ -534,13 +521,8 @@ def _setup_whatsapp_routes(app: FastAPI):
     _add_lifespan_to_app(app, whatsapp_lifespan)
 
 
-def _setup_daily_routes(app: FastAPI, dialin_enabled: bool = False):
-    """Set up Daily-specific routes.
-
-    Args:
-        app: FastAPI application instance
-        dialin_enabled: If True, adds /daily-dialin-webhook endpoint for PSTN dial-in handling
-    """
+def _setup_daily_routes(app: FastAPI, args: argparse.Namespace):
+    """Set up Daily-specific routes."""
 
     @app.get("/")
     async def create_room_and_start_agent():
@@ -557,6 +539,7 @@ def _setup_daily_routes(app: FastAPI, dialin_enabled: bool = False):
             # Start the bot in the background with empty body for GET requests
             bot_module = _get_bot_module()
             runner_args = DailyRunnerArguments(room_url=room_url, token=token)
+            runner_args.cli_args = args
             asyncio.create_task(bot_module.bot(runner_args))
             return RedirectResponse(room_url)
 
@@ -589,13 +572,13 @@ def _setup_daily_routes(app: FastAPI, dialin_enabled: bool = False):
 
         bot_module = _get_bot_module()
 
-        existing_room_url = os.getenv("DAILY_SAMPLE_ROOM_URL")
+        existing_room_url = os.getenv("DAILY_ROOM_URL")
 
         result = None
 
         # Configure room if:
         # 1. Explicitly requested via createDailyRoom in payload
-        # 2. Using pre-configured room from DAILY_SAMPLE_ROOM_URL env var
+        # 2. Using pre-configured room from DAILY_ROOM_URL env var
         if create_daily_room or existing_room_url:
             import aiohttp
 
@@ -640,12 +623,15 @@ def _setup_daily_routes(app: FastAPI, dialin_enabled: bool = False):
         else:
             runner_args = RunnerArguments(body=body)
 
+        # Update CLI args.
+        runner_args.cli_args = args
+
         # Start the bot in the background
         asyncio.create_task(bot_module.bot(runner_args))
 
         return result
 
-    if dialin_enabled:
+    if args.dialin:
 
         @app.post("/daily-dialin-webhook")
         async def handle_dialin_webhook(request: Request):
@@ -742,6 +728,7 @@ def _setup_daily_routes(app: FastAPI, dialin_enabled: bool = False):
                 token=room_config.token,
                 body=request_body.model_dump(),
             )
+            runner_args.cli_args = args
 
             asyncio.create_task(bot_module.bot(runner_args))
 
@@ -756,44 +743,44 @@ def _setup_daily_routes(app: FastAPI, dialin_enabled: bool = False):
             }
 
 
-def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str):
+def _setup_telephony_routes(app: FastAPI, args: argparse.Namespace):
     """Set up telephony-specific routes."""
     # XML response templates (Exotel doesn't use XML webhooks)
     XML_TEMPLATES = {
         "twilio": f"""<?xml version="1.0" encoding="UTF-8"?>
 <Response>
   <Connect>
-    <Stream url="wss://{proxy}/ws"></Stream>
+    <Stream url="wss://{args.proxy}/ws"></Stream>
   </Connect>
   <Pause length="40"/>
 </Response>""",
         "telnyx": f"""<?xml version="1.0" encoding="UTF-8"?>
 <Response>
   <Connect>
-    <Stream url="wss://{proxy}/ws" bidirectionalMode="rtp"></Stream>
+    <Stream url="wss://{args.proxy}/ws" bidirectionalMode="rtp"></Stream>
   </Connect>
   <Pause length="40"/>
 </Response>""",
         "plivo": f"""<?xml version="1.0" encoding="UTF-8"?>
 <Response>
-  <Stream bidirectional="true" keepCallAlive="true" contentType="audio/x-mulaw;rate=8000">wss://{proxy}/ws</Stream>
+  <Stream bidirectional="true" keepCallAlive="true" contentType="audio/x-mulaw;rate=8000">wss://{args.proxy}/ws</Stream>
 </Response>""",
     }
 
     @app.post("/")
     async def start_call():
         """Handle telephony webhook and return XML response."""
-        if transport_type == "exotel":
+        if args.transport == "exotel":
             # Exotel doesn't use POST webhooks - redirect to proper documentation
             logger.debug("POST Exotel endpoint - not used")
             return {
                 "error": "Exotel doesn't use POST webhooks",
-                "websocket_url": f"wss://{proxy}/ws",
+                "websocket_url": f"wss://{args.proxy}/ws",
                 "note": "Configure the WebSocket URL above in your Exotel App Bazaar Voicebot Applet",
             }
         else:
-            logger.debug(f"POST {transport_type.upper()} XML")
-            xml_content = XML_TEMPLATES.get(transport_type, "<Response></Response>")
+            logger.debug(f"POST {args.transport.upper()} XML")
+            xml_content = XML_TEMPLATES.get(args.transport, "<Response></Response>")
             return HTMLResponse(content=xml_content, media_type="application/xml")
 
     @app.websocket("/ws")
@@ -801,15 +788,15 @@ def _setup_telephony_routes(app: FastAPI, *, transport_type: str, proxy: str):
         """Handle WebSocket connections for telephony."""
         await websocket.accept()
         logger.debug("WebSocket connection accepted")
-        await _run_telephony_bot(websocket)
+        await _run_telephony_bot(websocket, args)
 
     @app.get("/")
     async def start_agent():
         """Simple status endpoint for telephony transports."""
-        return {"status": f"Bot started with {transport_type}"}
+        return {"status": f"Bot started with {args.transport}"}
 
 
-async def _run_daily_direct():
+async def _run_daily_direct(args: argparse.Namespace):
     """Run Daily bot with direct connection (no FastAPI server)."""
     try:
         from pipecat.runner.daily import configure
@@ -825,6 +812,7 @@ async def _run_daily_direct():
         # Direct connections have no request body, so use empty dict
         runner_args = DailyRunnerArguments(room_url=room_url, token=token)
         runner_args.handle_sigint = True
+        runner_args.cli_args = args
 
         # Get the bot module and run it directly
         bot_module = _get_bot_module()
@@ -872,29 +860,38 @@ def runner_port() -> int:
     return RUNNER_PORT
 
 
-def main():
+def main(parser: Optional[argparse.ArgumentParser] = None):
     """Start the Pipecat development runner.
 
     Parses command-line arguments and starts a FastAPI server configured
-    for the specified transport type. The runner will discover and run
-    any bot() function found in the current directory.
+    for the specified transport type.
+
+    The runner discovers and runs any ``bot(runner_args)`` function found in the
+    calling module.
 
     Command-line arguments:
+       - --host: Server host address (default: localhost) 879
+       - --port: Server port (default: 7860)
+       - -t/--transport: Transport type (daily, webrtc, twilio, telnyx, plivo, exotel)
+       - -x/--proxy: Public proxy hostname for telephony webhooks
+       - -d/--direct: Connect directly to Daily room (automatically sets transport to daily)
+       - -f/--folder: Path to downloads folder
+       - --dialin: Enable Daily PSTN dial-in webhook handling (requires Daily transport)
+       - --esp32: Enable SDP munging for ESP32 compatibility (requires --host with IP address)
+       - --whatsapp: Ensure requried WhatsApp environment variables are present
+       - -v/--verbose: Increase logging verbosity
 
     Args:
-        --host: Server host address (default: localhost)
-        --port: Server port (default: 7860)
-        -t/--transport: Transport type (daily, webrtc, twilio, telnyx, plivo, exotel)
-        -x/--proxy: Public proxy hostname for telephony webhooks
-        --esp32: Enable SDP munging for ESP32 compatibility (requires --host with IP address)
-        -d/--direct: Connect directly to Daily room (automatically sets transport to daily)
-        -v/--verbose: Increase logging verbosity
+        parser: Optional custom argument parser. If provided, default runner
+            arguments are added to it so bots can define their own CLI
+            arguments. Custom arguments should not conflict with the default
+            ones. Custom args are accessible via `runner_args.cli_args`.
 
-    The bot file must contain a `bot(runner_args)` function as the entry point.
     """
     global RUNNER_DOWNLOADS_FOLDER, RUNNER_HOST, RUNNER_PORT
 
-    parser = argparse.ArgumentParser(description="Pipecat Development Runner")
+    if not parser:
+        parser = argparse.ArgumentParser(description="Pipecat Development Runner")
     parser.add_argument("--host", type=str, default=RUNNER_HOST, help="Host address")
     parser.add_argument("--port", type=int, default=RUNNER_PORT, help="Port number")
     parser.add_argument(
@@ -905,13 +902,7 @@ def main():
         default="webrtc",
         help="Transport type",
     )
-    parser.add_argument("--proxy", "-x", help="Public proxy host name")
-    parser.add_argument(
-        "--esp32",
-        action="store_true",
-        default=False,
-        help="Enable SDP munging for ESP32 compatibility (requires --host with IP address)",
-    )
+    parser.add_argument("-x", "--proxy", help="Public proxy host name")
     parser.add_argument(
         "-d",
         "--direct",
@@ -921,13 +912,7 @@ def main():
     )
     parser.add_argument("-f", "--folder", type=str, help="Path to downloads folder")
     parser.add_argument(
-        "--verbose", "-v", action="count", default=0, help="Increase logging verbosity"
-    )
-    parser.add_argument(
-        "--whatsapp",
-        action="store_true",
-        default=False,
-        help="Ensure requried WhatsApp environment variables are present",
+        "-v", "--verbose", action="count", default=0, help="Increase logging verbosity"
     )
     parser.add_argument(
         "--dialin",
@@ -935,6 +920,18 @@ def main():
         default=False,
         help="Enable Daily PSTN dial-in webhook handling (requires Daily transport)",
     )
+    parser.add_argument(
+        "--esp32",
+        action="store_true",
+        default=False,
+        help="Enable SDP munging for ESP32 compatibility (requires --host with IP address)",
+    )
+    parser.add_argument(
+        "--whatsapp",
+        action="store_true",
+        default=False,
+        help="Ensure requried WhatsApp environment variables are present",
+    )
 
     args = parser.parse_args()
 
@@ -970,7 +967,7 @@ def main():
         print()
 
         # Run direct Daily connection
-        asyncio.run(_run_daily_direct())
+        asyncio.run(_run_daily_direct(args))
         return
 
     # Print startup message for server-based transports
@@ -1001,15 +998,7 @@ def main():
     RUNNER_PORT = args.port
 
     # Create the app with transport-specific setup
-    app = _create_server_app(
-        transport_type=args.transport,
-        host=args.host,
-        proxy=args.proxy,
-        esp32_mode=args.esp32,
-        whatsapp_enabled=args.whatsapp,
-        folder=args.folder,
-        dialin_enabled=args.dialin,
-    )
+    app = _create_server_app(args)
 
     # Run the server
     uvicorn.run(app, host=args.host, port=args.port)

src/pipecat/runner/types.py

@@ -10,6 +10,7 @@ These types are used by the development runner to pass transport-specific
 information to bot functions.
 """
 
+import argparse
 from dataclasses import dataclass, field
 from typing import Any, Dict, Optional
 
@@ -64,6 +65,7 @@ class RunnerArguments:
     handle_sigterm: bool = field(init=False, kw_only=True)
     pipeline_idle_timeout_secs: int = field(init=False, kw_only=True)
     body: Optional[Any] = field(default_factory=dict, kw_only=True)
+    cli_args: Optional[argparse.Namespace] = field(default=None, init=False, kw_only=True)
 
     def __post_init__(self):
         self.handle_sigint = False
@@ -106,3 +108,18 @@ class SmallWebRTCRunnerArguments(RunnerArguments):
     """
 
     webrtc_connection: Any
+
+
+@dataclass
+class LiveKitRunnerArguments(RunnerArguments):
+    """LiveKit transport session arguments for the runner.
+
+    Parameters:
+        room_name: LiveKit room name to join
+        token: Authentication token for the room
+        body: Additional request data
+    """
+
+    room_name: str
+    url: str
+    token: Optional[str] = None

src/pipecat/runner/utils.py

@@ -39,6 +39,7 @@ from loguru import logger
 
 from pipecat.runner.types import (
     DailyRunnerArguments,
+    LiveKitRunnerArguments,
     SmallWebRTCRunnerArguments,
     WebSocketRunnerArguments,
 )
@@ -95,6 +96,9 @@ def _detect_transport_type_from_message(message_data: dict) -> str:
 async def parse_telephony_websocket(websocket: WebSocket):
     """Parse telephony WebSocket messages and return transport type and call data.
 
+    Args:
+        websocket: FastAPI WebSocket connection from telephony provider.
+
     Returns:
         tuple: (transport_type: str, call_data: dict)
 
@@ -135,6 +139,9 @@ async def parse_telephony_websocket(websocket: WebSocket):
                 "to": str,
             }
 
+    Raises:
+        ValueError: If WebSocket closes before sending any messages.
+
     Example usage::
 
         transport_type, call_data = await parse_telephony_websocket(websocket)
@@ -142,25 +149,31 @@ async def parse_telephony_websocket(websocket: WebSocket):
             user_id = call_data["body"]["user_id"]
     """
     # Read first two messages
-    start_data = websocket.iter_text()
+    message_stream = websocket.iter_text()
+    first_message = {}
+    second_message = {}
 
     try:
-        # First message
-        first_message_raw = await start_data.__anext__()
+        # First message - required
+        first_message_raw = await message_stream.__anext__()
         logger.trace(f"First message: {first_message_raw}")
-        try:
-            first_message = json.loads(first_message_raw)
-        except json.JSONDecodeError:
-            first_message = {}
+        first_message = json.loads(first_message_raw) if first_message_raw else {}
+    except json.JSONDecodeError:
+        pass
+    except StopAsyncIteration:
+        raise ValueError("WebSocket closed before receiving telephony handshake messages")
 
-        # Second message
-        second_message_raw = await start_data.__anext__()
+    try:
+        # Second message - optional, some providers may only send one
+        second_message_raw = await message_stream.__anext__()
         logger.trace(f"Second message: {second_message_raw}")
-        try:
-            second_message = json.loads(second_message_raw)
-        except json.JSONDecodeError:
-            second_message = {}
+        second_message = json.loads(second_message_raw) if second_message_raw else {}
+    except json.JSONDecodeError:
+        pass
+    except StopAsyncIteration:
+        logger.warning("Only received one WebSocket message, expected two")
 
+    try:
         # Try auto-detection on both messages
         detected_type_first = _detect_transport_type_from_message(first_message)
         detected_type_second = _detect_transport_type_from_message(second_message)
@@ -568,6 +581,17 @@ async def create_transport(
         return await _create_telephony_transport(
             runner_args.websocket, params, transport_type, call_data
         )
+    elif isinstance(runner_args, LiveKitRunnerArguments):
+        params = _get_transport_params("livekit", transport_params)
+
+        from pipecat.transports.livekit.transport import LiveKitTransport
+
+        return LiveKitTransport(
+            runner_args.url,
+            runner_args.token,
+            runner_args.room_name,
+            params=params,
+        )
 
     else:
         raise ValueError(f"Unsupported runner arguments type: {type(runner_args)}")

src/pipecat/serializers/base_serializer.py

@@ -6,12 +6,22 @@
 
 """Frame serialization interfaces for Pipecat."""
 
-from abc import ABC, abstractmethod
+from abc import abstractmethod
+from typing import Optional
 
-from pipecat.frames.frames import Frame, StartFrame
+from pydantic import BaseModel
+
+import pipecat.processors.frameworks.rtvi.models as RTVI
+from pipecat.frames.frames import (
+    Frame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
+    StartFrame,
+)
+from pipecat.utils.base_object import BaseObject
 
 
-class FrameSerializer(ABC):
+class FrameSerializer(BaseObject):
     """Abstract base class for frame serialization implementations.
 
     Defines the interface for converting frames to/from serialized formats
@@ -19,6 +29,46 @@ class FrameSerializer(ABC):
     serialize/deserialize methods.
     """
 
+    class InputParams(BaseModel):
+        """Base configuration parameters for FrameSerializer.
+
+        Parameters:
+            ignore_rtvi_messages: Whether to ignore RTVI protocol messages during serialization.
+                Defaults to True to prevent RTVI messages from being sent to external transports.
+        """
+
+        ignore_rtvi_messages: bool = True
+
+    def __init__(self, params: Optional[InputParams] = None, **kwargs):
+        """Initialize the FrameSerializer.
+
+        Args:
+            params: Configuration parameters.
+            **kwargs: Additional arguments passed to BaseObject (e.g., name).
+        """
+        super().__init__(**kwargs)
+        self._params = params or FrameSerializer.InputParams()
+
+    def should_ignore_frame(self, frame: Frame) -> bool:
+        """Check if a frame should be ignored during serialization.
+
+        This method filters out RTVI protocol messages when ignore_rtvi_messages is enabled.
+        Subclasses can override this to add additional filtering logic.
+
+        Args:
+            frame: The frame to check.
+
+        Returns:
+            True if the frame should be ignored, False otherwise.
+        """
+        if (
+            self._params.ignore_rtvi_messages
+            and isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame))
+            and frame.message.get("label") == RTVI.MESSAGE_LABEL
+        ):
+            return True
+        return False
+
     async def setup(self, frame: StartFrame):
         """Initialize the serializer with startup configuration.
 

src/pipecat/serializers/exotel.py

@@ -11,7 +11,6 @@ import json
 from typing import Optional
 
 from loguru import logger
-from pydantic import BaseModel
 
 from pipecat.audio.dtmf.types import KeypadEntry
 from pipecat.audio.utils import create_stream_resampler
@@ -39,12 +38,13 @@ class ExotelFrameSerializer(FrameSerializer):
         https://support.exotel.com/support/solutions/articles/3000108630-working-with-the-stream-and-voicebot-applet
     """
 
-    class InputParams(BaseModel):
+    class InputParams(FrameSerializer.InputParams):
         """Configuration parameters for ExotelFrameSerializer.
 
         Parameters:
             exotel_sample_rate: Sample rate used by Exotel, defaults to 8000 Hz.
             sample_rate: Optional override for pipeline input sample rate.
+            ignore_rtvi_messages: Inherited from base FrameSerializer, defaults to True.
         """
 
         exotel_sample_rate: int = 8000
@@ -60,9 +60,10 @@ class ExotelFrameSerializer(FrameSerializer):
             call_sid: The associated Exotel Call SID (optional, not used in this implementation).
             params: Configuration parameters.
         """
+        super().__init__(params or ExotelFrameSerializer.InputParams())
+
         self._stream_sid = stream_sid
         self._call_sid = call_sid
-        self._params = params or ExotelFrameSerializer.InputParams()
 
         self._exotel_sample_rate = self._params.exotel_sample_rate
         self._sample_rate = 0  # Pipeline input rate
@@ -113,6 +114,8 @@ class ExotelFrameSerializer(FrameSerializer):
 
             return json.dumps(answer)
         elif isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
+            if self.should_ignore_frame(frame):
+                return None
             return json.dumps(frame.message)
 
         return None

src/pipecat/serializers/genesys.py

@@ -0,0 +1,963 @@
+"""Genesys AudioHook Serializer for Pipecat.
+
+This module provides a serializer for integrating Pipecat pipelines with
+Genesys Cloud Contact Center via the AudioHook protocol.
+
+Features:
+- Bidirectional audio streaming (PCMU μ-law at 8kHz)
+- Automatic protocol handshake handling (open/opened, close/closed, ping/pong)
+- Input/output variables for Architect flow integration
+- DTMF event support
+- Barge-in (interruption) events
+- Pause/resume support for hold scenarios (optional)
+
+Protocol Reference:
+- https://developer.genesys.cloud/devapps/audiohook
+
+Audio Format:
+- PCMU (μ-law) at 8kHz sample rate (preferred)
+- L16 (16-bit linear PCM) at 8kHz also supported
+- Mono (external channel) or Stereo (external on left, internal on right)
+"""
+
+import json
+import uuid
+from datetime import timedelta
+from enum import Enum
+from typing import Any, Dict, List, Optional
+
+from loguru import logger
+
+from pipecat.audio.dtmf.types import KeypadEntry
+from pipecat.audio.resamplers.soxr_stream_resampler import SOXRStreamAudioResampler
+from pipecat.audio.utils import pcm_to_ulaw, ulaw_to_pcm
+from pipecat.frames.frames import (
+    AudioRawFrame,
+    CancelFrame,
+    EndFrame,
+    Frame,
+    InputAudioRawFrame,
+    InputDTMFFrame,
+    InterruptionFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
+    StartFrame,
+)
+from pipecat.serializers.base_serializer import FrameSerializer
+
+
+class AudioHookMessageType(str, Enum):
+    """AudioHook protocol message types."""
+
+    OPEN = "open"
+    OPENED = "opened"
+    CLOSE = "close"
+    CLOSED = "closed"
+    PAUSE = "pause"
+    RESUMED = "resumed"
+    PING = "ping"
+    PONG = "pong"
+    UPDATE = "update"
+    EVENT = "event"
+    ERROR = "error"
+    DISCONNECT = "disconnect"
+
+
+class AudioHookChannel(str, Enum):
+    """AudioHook audio channel configuration."""
+
+    EXTERNAL = "external"  # Customer audio only (mono)
+    INTERNAL = "internal"  # Agent audio only (mono)
+    BOTH = "both"  # Stereo: external=left, internal=right
+
+
+class AudioHookMediaFormat(str, Enum):
+    """Supported audio formats."""
+
+    PCMU = "PCMU"  # μ-law, 8kHz
+    L16 = "L16"  # 16-bit linear PCM, 8kHz
+
+
+class GenesysAudioHookSerializer(FrameSerializer):
+    """Serializer for Genesys AudioHook WebSocket protocol.
+
+    This serializer handles converting between Pipecat frames and Genesys
+    AudioHook protocol messages. It supports:
+
+    - Bidirectional audio streaming (PCMU at 8kHz)
+    - Automatic protocol handshake (open/opened, close/closed, ping/pong)
+    - Session lifecycle management with pause/resume support
+    - Custom input/output variables for Architect flow integration
+    - DTMF event handling
+    - Barge-in events for interruption support
+
+    The AudioHook protocol uses:
+    - Text WebSocket frames for JSON control messages
+    - Binary WebSocket frames for audio data
+
+    Example usage:
+        ```python
+        serializer = GenesysAudioHookSerializer(
+            params=GenesysAudioHookSerializer.InputParams(
+                channel=AudioHookChannel.EXTERNAL,
+                supported_languages=["en-US", "es-ES"],
+                selected_language="en-US",
+            )
+        )
+
+        # Use with FastAPI WebSocket transport
+        transport = FastAPIWebsocketTransport(
+            websocket=websocket,
+            params=FastAPIWebsocketParams(
+                audio_in_enabled=True,
+                audio_out_enabled=True,
+                serializer=serializer,
+                audio_out_fixed_packet_size=1600,  # Important: prevents 429 rate limiting from Genesys
+            ),
+        )
+
+        # Access call information after connection
+        participant = serializer.participant  # ani, dnis, etc.
+        input_vars = serializer.input_variables  # Custom vars from Architect
+
+        # Set output variables to return to Architect
+        serializer.set_output_variables({"intent": "billing", "resolved": True})
+        ```
+
+    Attributes:
+        PROTOCOL_VERSION: The AudioHook protocol version (currently "2").
+    """
+
+    PROTOCOL_VERSION = "2"
+
+    class InputParams(FrameSerializer.InputParams):
+        """Configuration parameters for GenesysAudioHookSerializer.
+
+        Attributes:
+            genesys_sample_rate: Sample rate used by Genesys (default: 8000 Hz).
+            sample_rate: Optional override for pipeline input sample rate.
+            channel: Which audio channels to process (external, internal, both).
+            media_format: Audio format (PCMU or L16).
+            process_external: Whether to process external (customer) audio.
+            process_internal: Whether to process internal (agent) audio.
+            supported_languages: List of language codes the bot supports (e.g., ["en-US", "es-ES"]).
+            selected_language: Default language code to use.
+            start_paused: Whether to start the session in paused state.
+            ignore_rtvi_messages: Inherited from base FrameSerializer, defaults to True.
+        """
+
+        genesys_sample_rate: int = 8000
+        sample_rate: Optional[int] = None
+        channel: AudioHookChannel = AudioHookChannel.EXTERNAL
+        media_format: AudioHookMediaFormat = AudioHookMediaFormat.PCMU
+        process_external: bool = True
+        process_internal: bool = False
+        supported_languages: Optional[List[str]] = None
+        selected_language: Optional[str] = None
+        start_paused: bool = False
+
+    def __init__(
+        self,
+        params: Optional[InputParams] = None,
+        **kwargs,
+    ):
+        """Initialize the GenesysAudioHookSerializer.
+
+        Args:
+            params: Configuration parameters.
+            **kwargs: Additional arguments passed to BaseObject (e.g., name).
+        """
+        super().__init__(params or GenesysAudioHookSerializer.InputParams(), **kwargs)
+
+        self._genesys_sample_rate = self._params.genesys_sample_rate
+        self._sample_rate = 0  # Pipeline input rate, set in setup()
+        self._session_id = str(uuid.uuid4())
+
+        # Use Pipecat's official resampler if needed (SOXR)
+        # Only used for TTS output (16kHz → 8kHz), input goes without resampling
+        self._input_resampler = SOXRStreamAudioResampler()
+        self._output_resampler = SOXRStreamAudioResampler()
+
+        # Protocol state
+        self._client_seq = 0
+        self._server_seq = 0
+        self._is_open = False
+        self._is_paused = False
+        self._position = timedelta(0)
+
+        # Session metadata
+        self._conversation_id: Optional[str] = None
+        self._participant: Optional[Dict[str, Any]] = None
+        self._custom_config: Optional[Dict[str, Any]] = None
+        self._media_info: Optional[List[Dict[str, Any]]] = None
+        self._input_variables: Optional[Dict[str, Any]] = None  # Custom input from Genesys
+        self._output_variables: Optional[Dict[str, Any]] = None  # Custom output to Genesys
+
+        # Event handlers
+        self._register_event_handler("on_open")
+        self._register_event_handler("on_close")
+        self._register_event_handler("on_ping")
+        self._register_event_handler("on_pause")
+        self._register_event_handler("on_update")
+        self._register_event_handler("on_error")
+        self._register_event_handler("on_dtmf")
+
+    @property
+    def session_id(self) -> str:
+        """Get the Genesys AudioHook session ID generated by the serializer."""
+        return self._session_id
+
+    @property
+    def conversation_id(self) -> Optional[str]:
+        """Get the Genesys conversation ID."""
+        return self._conversation_id
+
+    @property
+    def is_open(self) -> bool:
+        """Check if the AudioHook session is open."""
+        return self._is_open
+
+    @property
+    def is_paused(self) -> bool:
+        """Check if audio streaming is paused."""
+        return self._is_paused
+
+    @property
+    def participant(self) -> Optional[Dict[str, Any]]:
+        """Get participant info (ani, dnis, etc.) from the open message."""
+        return self._participant
+
+    @property
+    def input_variables(self) -> Optional[Dict[str, Any]]:
+        """Get custom input variables from the open message."""
+        return self._input_variables
+
+    @property
+    def output_variables(self) -> Optional[Dict[str, Any]]:
+        """Get custom output variables to send back to Genesys."""
+        return self._output_variables
+
+    def set_output_variables(self, variables: Dict[str, Any]) -> None:
+        """Set custom output variables to send back to Genesys on close.
+
+        These variables will be included in the 'closed' response when Genesys
+        closes the connection, making them available in the Architect flow.
+
+        Args:
+            variables: Dictionary of custom variables to send to Genesys.
+
+        Example:
+            ```python
+            # During the conversation, collect data and set it
+            serializer.set_output_variables({
+                "intent": "billing_inquiry",
+                "customer_verified": True,
+                "summary": "Customer asked about their bill",
+                "transfer_to": "billing_queue"
+            })
+            ```
+        """
+        self._output_variables = variables
+        logger.debug(f"Output variables set: {variables}")
+
+    async def setup(self, frame: StartFrame):
+        """Sets up the serializer with pipeline configuration.
+
+        Args:
+            frame: The StartFrame containing pipeline configuration.
+        """
+        self._sample_rate = self._params.sample_rate or frame.audio_in_sample_rate
+        logger.debug(f"GenesysAudioHookSerializer setup with sample_rate={self._sample_rate}")
+
+    def _format_position(self, position: timedelta) -> str:
+        """Format a timedelta as ISO 8601 duration string.
+
+        Args:
+            position: The timedelta to format.
+
+        Returns:
+            ISO 8601 duration string (e.g., "PT1.5S").
+        """
+        total_seconds = position.total_seconds()
+        return f"PT{total_seconds:.3f}S"
+
+    def _parse_position(self, position_str: str) -> timedelta:
+        """Parse an ISO 8601 duration string to timedelta.
+
+        Args:
+            position_str: ISO 8601 duration string (e.g., "PT1.5S").
+
+        Returns:
+            Corresponding timedelta.
+        """
+        # Simple parser for PT#S or PT#.#S format
+        if position_str.startswith("PT") and position_str.endswith("S"):
+            try:
+                seconds = float(position_str[2:-1])
+                return timedelta(seconds=seconds)
+            except ValueError:
+                pass
+        return timedelta(0)
+
+    def _next_server_seq(self) -> int:
+        """Get the next server sequence number."""
+        self._server_seq += 1
+        return self._server_seq
+
+    def _create_message(
+        self,
+        msg_type: AudioHookMessageType,
+        parameters: Optional[Dict[str, Any]] = None,
+        include_position: bool = True,
+    ) -> Dict[str, Any]:
+        """Create a protocol message with common fields.
+
+        Based on the Genesys AudioHook protocol, responses include:
+        - seq: Server's sequence number (incremented per message)
+        - clientseq: Echo of the client's last sequence number
+
+        Args:
+            msg_type: The message type.
+            parameters: Optional parameters object.
+            include_position: Whether to include position field.
+
+        Returns:
+            The message dictionary.
+        """
+        seq = self._next_server_seq()
+        msg = {
+            "version": self.PROTOCOL_VERSION,
+            "type": msg_type.value,
+            "seq": seq,
+            "clientseq": self._client_seq,
+            "id": self._session_id,
+        }
+
+        if include_position:
+            msg["position"] = self._format_position(self._position)
+
+        msg["parameters"] = parameters if parameters is not None else {}
+
+        return msg
+
+    def create_opened_response(
+        self,
+        start_paused: bool = False,
+        supported_languages: Optional[List[str]] = None,
+        selected_language: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Create an 'opened' response message for the client.
+
+        This should be sent in response to an 'open' message from Genesys.
+
+        Args:
+            start_paused: Whether to start the session paused.
+            supported_languages: List of supported language codes.
+            selected_language: The selected language code.
+
+        Returns:
+            Dictionary of the opened response message.
+        """
+        # Build channels list based on configuration
+        channels: list[str] = []
+
+        if self._params.channel == AudioHookChannel.EXTERNAL:
+            channels = ["external"]
+        elif self._params.channel == AudioHookChannel.INTERNAL:
+            channels = ["internal"]
+        elif self._params.channel == AudioHookChannel.BOTH:
+            channels = ["external", "internal"]
+
+        parameters = {
+            "startPaused": start_paused,
+            "media": [
+                {
+                    "type": "audio",
+                    "format": self._params.media_format.value,
+                    "channels": channels,
+                    "rate": self._genesys_sample_rate,
+                }
+            ],
+        }
+
+        if supported_languages:
+            parameters["supportedLanguages"] = supported_languages
+        if selected_language:
+            parameters["selectedLanguage"] = selected_language
+
+        msg = self._create_message(
+            AudioHookMessageType.OPENED,
+            parameters=parameters,
+            include_position=False,  # opened doesn't need position
+        )
+
+        self._is_open = True
+
+        logger.debug(f"AudioHook session opened: {self._session_id}")
+
+        return msg
+
+    def create_closed_response(
+        self,
+        output_variables: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """Create a 'closed' response message.
+
+        This should be sent in response to a 'close' message from Genesys.
+
+        Args:
+            output_variables: Optional custom variables to pass back to Genesys.
+                These will be available in the Architect flow after the AudioHook
+                action completes.
+
+        Returns:
+            Dictionary of the closed response message.
+
+        Example:
+            ```python
+            # Pass custom data back to Genesys
+            serializer.create_closed_response(
+                output_variables={
+                    "intent": "billing_inquiry",
+                    "customer_verified": True,
+                    "summary": "Customer asked about their bill"
+                }
+            )
+            ```
+        """
+        parameters: Optional[Dict[str, Any]] = None
+
+        if output_variables:
+            parameters = {"outputVariables": output_variables}
+
+        msg = self._create_message(
+            AudioHookMessageType.CLOSED,
+            parameters=parameters,
+        )
+
+        self._is_open = False
+        logger.debug(f"AudioHook session closed: {self._session_id}")
+
+        return msg
+
+    def create_pong_response(self) -> Dict[str, Any]:
+        """Create a 'pong' response message.
+
+        This should be sent in response to a 'ping' message from Genesys.
+
+        Returns:
+            Dictionary of the pong response message.
+        """
+        msg = self._create_message(AudioHookMessageType.PONG)
+        return msg
+
+    def create_resumed_response(self) -> Dict[str, Any]:
+        """Create a 'resumed' response message.
+
+        This should be sent in response to a 'pause' message when ready to resume.
+
+        Returns:
+            Dictionary of the resumed response message.
+        """
+        msg = self._create_message(AudioHookMessageType.RESUMED)
+
+        self._is_paused = False
+        logger.debug(f"AudioHook session resumed: {self._session_id}")
+
+        return msg
+
+    def create_barge_in_event(self) -> Dict[str, Any]:
+        """Create a barge-in event message.
+
+        This notifies Genesys Cloud that the user has interrupted the bot's
+        audio output. Genesys will stop any queued audio playback.
+
+        Returns:
+            Dictionary of the barge-in event message.
+        """
+        msg = self._create_message(
+            AudioHookMessageType.EVENT,
+            parameters={"entities": [{"type": "barge_in", "data": {}}]},
+        )
+
+        logger.debug("🔇 Barge-in event sent to Genesys")
+
+        return msg
+
+    def create_disconnect_message(
+        self,
+        reason: str = "completed",
+        action: str = "transfer",
+        output_variables: Optional[Dict[str, Any]] = None,
+        info: Optional[str] = None,
+    ) -> Dict[str, Any]:
+        """Create a 'disconnect' message to initiate session termination.
+
+        Args:
+            reason: Disconnect reason (e.g., "completed", "error").
+            action: Action to take ("transfer" to agent, "finished" if completed).
+            output_variables: Custom output variables to pass back to Genesys.
+            info: Optional additional information.
+
+        Returns:
+            Dictionary of the disconnect message.
+        """
+        parameters: Dict[str, Any] = {"reason": reason}
+
+        # Build outputVariables
+        out_vars = {"action": action}
+        if output_variables:
+            out_vars.update(output_variables)
+        parameters["outputVariables"] = out_vars
+
+        if info:
+            parameters["info"] = info
+
+        msg = self._create_message(
+            AudioHookMessageType.DISCONNECT,
+            parameters=parameters,
+        )
+
+        logger.debug(f"AudioHook disconnect: reason={reason}, action={action}")
+        return msg
+
+    def create_error_message(
+        self,
+        code: int,
+        message: str,
+        retryable: bool = False,
+    ) -> Dict[str, Any]:
+        """Create an 'error' message.
+
+        Args:
+            code: Error code.
+            message: Error message.
+            retryable: Whether the operation can be retried.
+
+        Returns:
+            Dictionary of the error message.
+        """
+        parameters = {
+            "code": code,
+            "message": message,
+            "retryable": retryable,
+        }
+
+        msg = self._create_message(
+            AudioHookMessageType.ERROR,
+            parameters=parameters,
+        )
+
+        logger.error(f"AudioHook error: {code} - {message}")
+        return msg
+
+    async def serialize(self, frame: Frame) -> str | bytes | None:
+        """Serializes a Pipecat frame to Genesys AudioHook format.
+
+        Handles conversion of various frame types to AudioHook messages:
+        - AudioRawFrame -> Binary PCMU audio data (resampled to 8kHz)
+        - EndFrame/CancelFrame -> Disconnect message (JSON)
+        - InterruptionFrame -> Barge-in event (JSON)
+        - OutputTransportMessageFrame -> Pass-through JSON
+
+        Args:
+            frame: The Pipecat frame to serialize.
+
+        Returns:
+            Serialized data as string (JSON) or bytes (audio), or None if
+            the frame type is not handled or session is not open.
+        """
+        if isinstance(frame, (EndFrame, CancelFrame)):
+            return json.dumps(
+                self.create_disconnect_message(
+                    output_variables=self.output_variables, reason="completed"
+                )
+            )
+
+        elif isinstance(frame, AudioRawFrame):
+            if not self._is_open or self._is_paused:
+                return None
+
+            data = frame.audio
+
+            # Convert PCM to μ-law at 8kHz for Genesys
+            if self._params.media_format == AudioHookMediaFormat.PCMU:
+                serialized_data = await pcm_to_ulaw(
+                    data,
+                    frame.sample_rate,
+                    self._genesys_sample_rate,
+                    self._output_resampler,
+                )
+            else:
+                # L16 format - just resample if needed
+                logger.warning("L16 format not yet fully implemented")
+                return None
+
+            if serialized_data is None or len(serialized_data) == 0:
+                return None
+
+            return bytes(serialized_data)
+
+        elif isinstance(frame, InterruptionFrame):
+            return json.dumps(self.create_barge_in_event())
+
+        elif isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
+            # Filter out RTVI messages using base class method
+            if self.should_ignore_frame(frame):
+                return None
+            # Only pass through AudioHook protocol messages (those with "version" field)
+            # Filter out RTVI and other non-AudioHook messages
+            if isinstance(frame.message, dict) and "version" in frame.message:
+                return json.dumps(frame.message)
+            else:
+                # Not an AudioHook message, ignore
+                return None
+
+        # Ignore other frames - we don't need to process them here
+        return None
+
+    async def deserialize(self, data: str | bytes) -> Frame | None:
+        """Deserializes Genesys AudioHook data to Pipecat frames.
+
+        Handles:
+        - Binary data -> InputAudioRawFrame (converted from PCMU to PCM)
+        - JSON 'open' -> OutputTransportMessageUrgentFrame with 'opened' response
+        - JSON 'close' -> OutputTransportMessageUrgentFrame with 'closed' response
+        - JSON 'ping' -> OutputTransportMessageUrgentFrame with 'pong' response
+        - JSON 'pause' -> Sets is_paused=True, returns None
+        - JSON 'dtmf' -> InputDTMFFrame
+        - JSON 'update' -> Updates participant info, returns None
+        - JSON 'error' -> Logs error, returns None
+
+        Protocol responses (opened, closed, pong) are returned as urgent frames
+        to be sent immediately through the transport.
+
+        Args:
+            data: The raw WebSocket data from Genesys (binary audio or JSON text).
+
+        Returns:
+            A Pipecat frame to process, or None if handled internally.
+        """
+        # Binary data = audio
+        if isinstance(data, bytes):
+            return await self._deserialize_audio(data)
+
+        # Text data = JSON control message
+        try:
+            message = json.loads(data)
+        except json.JSONDecodeError as e:
+            logger.error(f"Failed to parse AudioHook message: {e}")
+            return None
+
+        return await self._handle_control_message(message)
+
+    async def _deserialize_audio(self, data: bytes) -> Frame | None:
+        """Deserialize binary audio data to an InputAudioRawFrame.
+
+        Args:
+            data: Raw audio bytes (PCMU or L16).
+
+        Returns:
+            InputAudioRawFrame with PCM audio at pipeline sample rate.
+        """
+        if not self._is_open or self._is_paused:
+            return None
+
+        audio_data = data
+        original_len = len(data)
+
+        # If Genesys sends stereo audio (BOTH channels), extract only the external channel (left)
+        # Stereo audio comes interleaved: [L0, R0, L1, R1, ...]
+        if self._params.channel == AudioHookChannel.BOTH and len(data) > 0:
+            # For PCMU, each sample is 1 byte
+            # Extract only bytes at even positions (left channel = external)
+            audio_data = bytes(data[i] for i in range(0, len(data), 2))
+            logger.debug(
+                f"🔊 Stereo audio: {original_len} bytes → {len(audio_data)} bytes (external channel)"
+            )
+
+        if self._params.media_format == AudioHookMediaFormat.PCMU:
+            # Convert μ-law at 8kHz to PCM at pipeline rate
+            deserialized_data = await ulaw_to_pcm(
+                audio_data,
+                self._genesys_sample_rate,
+                self._sample_rate,
+                self._input_resampler,
+            )
+        else:
+            # L16 format
+            logger.warning("L16 format not yet fully implemented")
+            return None
+
+        if deserialized_data is None or len(deserialized_data) == 0:
+            return None
+
+        # Always use mono for STT - ElevenLabs expects single channel
+        num_channels = 1
+
+        audio_frame = InputAudioRawFrame(
+            audio=deserialized_data,
+            num_channels=num_channels,
+            sample_rate=self._sample_rate,
+        )
+
+        return audio_frame
+
+    async def _handle_control_message(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle a JSON control message from Genesys.
+
+        Args:
+            message: Parsed JSON message.
+
+        Returns:
+            Frame if the message should be passed to the pipeline, None otherwise.
+        """
+        msg_type = message.get("type", "")
+        self._client_seq = message.get("seq", 0)
+
+        # Update position if provided
+        if "position" in message:
+            self._position = self._parse_position(message["position"])
+
+        if msg_type == AudioHookMessageType.OPEN.value:
+            return await self._handle_open(message)
+
+        elif msg_type == AudioHookMessageType.CLOSE.value:
+            return await self._handle_close(message)
+
+        elif msg_type == AudioHookMessageType.PING.value:
+            return await self._handle_ping(message)
+
+        elif msg_type == AudioHookMessageType.PAUSE.value:
+            return await self._handle_pause(message)
+
+        elif msg_type == AudioHookMessageType.UPDATE.value:
+            return await self._handle_update(message)
+
+        elif msg_type == AudioHookMessageType.ERROR.value:
+            return await self._handle_error(message)
+
+        elif msg_type == "dtmf":
+            return await self._handle_dtmf(message)
+
+        elif msg_type == "playback_started":
+            logger.debug("Playback started (from Genesys)")
+            return None
+
+        elif msg_type == "playback_completed":
+            logger.debug("Playback completed (from Genesys)")
+            return None
+        else:
+            logger.warning(f"Unknown AudioHook message type: {msg_type}")
+            return None
+
+    async def _handle_open(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle an 'open' message from Genesys.
+
+        This initializes the session with metadata from Genesys Cloud and
+        automatically responds with an 'opened' message using the configured
+        InputParams (supported_languages, selected_language, start_paused).
+
+        Extracts and stores:
+        - session_id: The AudioHook session identifier
+        - conversation_id: The Genesys conversation ID
+        - participant: Caller info (ani, dnis, etc.)
+        - input_variables: Custom variables from Architect flow
+        - media_info: Audio configuration from Genesys
+
+        Args:
+            message: The open message from Genesys.
+
+        Returns:
+            OutputTransportMessageUrgentFrame with the 'opened' response.
+        """
+        self._session_id = message.get("id", str(uuid.uuid4()))
+
+        params = message.get("parameters", {})
+        self._conversation_id = params.get("conversationId")
+        self._participant = params.get("participant")
+        self._custom_config = params.get("customConfig")
+        self._media_info = params.get("media")  # This is a list of media objects
+        self._input_variables = params.get("inputVariables")  # Custom vars from Genesys
+
+        # Extract media configuration if present
+        # media is a list like: [{"type": "audio", "format": "PCMU", "channels": ["external"], "rate": 8000}]
+        media_list = self._media_info
+        if media_list and isinstance(media_list, list) and len(media_list) > 0:
+            audio_media: Dict[str, Any] = media_list[0]  # Get first media entry
+            channels = audio_media.get("channels", [])
+            logger.debug(
+                f"📡 Genesys audio config: format={audio_media.get('format')}, channels={channels}, rate={audio_media.get('rate')}"
+            )
+            # channels is a list like ["external"] or ["external", "internal"]
+            if isinstance(channels, list):
+                if "external" in channels and "internal" in channels:
+                    self._params.channel = AudioHookChannel.BOTH
+                    logger.debug("📡 Stereo mode: extracting external channel")
+                elif "external" in channels:
+                    self._params.channel = AudioHookChannel.EXTERNAL
+                    logger.debug("📡 Mono mode: external channel")
+                elif "internal" in channels:
+                    self._params.channel = AudioHookChannel.INTERNAL
+                    logger.debug("📡 Mono mode: internal channel")
+
+        # Log participant info for debugging
+        ani = self._participant.get("ani", "unknown") if self._participant else "unknown"
+        logger.info(
+            f"AudioHook open request: session={self._session_id}, "
+            f"conversation={self._conversation_id}, ani={ani}"
+        )
+
+        await self._call_event_handler("on_open", message)
+
+        return OutputTransportMessageUrgentFrame(
+            message=self.create_opened_response(
+                start_paused=self._params.start_paused,
+                supported_languages=self._params.supported_languages,
+                selected_language=self._params.selected_language,
+            )
+        )
+
+    async def _handle_close(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle a 'close' message from Genesys.
+
+        Automatically responds with a 'closed' message. If output_variables
+        were set via set_output_variables(), they will be included in the
+        response and made available in the Architect flow.
+
+        Args:
+            message: The close message from Genesys.
+
+        Returns:
+            OutputTransportMessageUrgentFrame with the closed response
+            (includes outputVariables if set).
+        """
+        params = message.get("parameters", {})
+        reason = params.get("reason", "unknown")
+
+        logger.info(f"🔴 Genesys closed the connection: {reason}")
+
+        self._is_open = False
+
+        logger.info(f"Sending closed response to Genesys...")
+
+        await self._call_event_handler("on_close", message)
+
+        # Return as urgent frame to be sent through pipeline immediately
+        # Include any output variables that were set during the session
+        return OutputTransportMessageUrgentFrame(
+            message=self.create_closed_response(output_variables=self._output_variables)
+        )
+
+    async def _handle_ping(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle a 'ping' message from Genesys.
+
+        Automatically responds with a 'pong' message to maintain the connection.
+
+        Args:
+            message: The ping message from Genesys.
+
+        Returns:
+            OutputTransportMessageUrgentFrame with pong response.
+        """
+        logger.info(f"Sending pong response to Genesys...")
+
+        await self._call_event_handler("on_ping", message)
+
+        # Return as urgent frame to be sent through pipeline immediately
+        return OutputTransportMessageUrgentFrame(message=self.create_pong_response())
+
+    async def _handle_pause(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle a 'pause' message from Genesys.
+
+        This is used when audio streaming is temporarily suspended
+        (e.g., during hold).
+
+        Args:
+            message: The pause message.
+
+        Returns:
+            None (response should be sent via create_resumed_response()).
+        """
+        params = message.get("parameters", {})
+        reason = params.get("reason", "unknown")
+
+        logger.info(f"AudioHook pause request: reason={reason}")
+
+        self._is_paused = True
+
+        await self._call_event_handler("on_pause", message)
+
+        # Note: Application should call create_resumed_response() when ready
+        return None
+
+    async def _handle_update(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle an 'update' message from Genesys.
+
+        Updates may include changes to participants or configuration.
+
+        Args:
+            message: The update message.
+
+        Returns:
+            None.
+        """
+        params = message.get("parameters", {})
+
+        if "participant" in params:
+            self._participant = params["participant"]
+
+        logger.debug(f"AudioHook update received: {params}")
+
+        await self._call_event_handler("on_update", message)
+
+        return None
+
+    async def _handle_error(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle an 'error' message from Genesys.
+
+        Args:
+            message: The error message.
+
+        Returns:
+            None.
+        """
+        params = message.get("parameters", {})
+        code = params.get("code", 0)
+        error_msg = params.get("message", "Unknown error")
+
+        logger.error(f"AudioHook error from Genesys: {code} - {error_msg}")
+
+        await self._call_event_handler("on_error", message)
+
+        return None
+
+    async def _handle_dtmf(self, message: Dict[str, Any]) -> Frame | None:
+        """Handle a 'dtmf' message from Genesys.
+
+        DTMF (Dual-Tone Multi-Frequency) events are sent when the user
+        presses keys on their phone keypad.
+
+        Args:
+            message: The DTMF message.
+
+        Returns:
+            InputDTMFFrame with the pressed digit.
+        """
+        params = message.get("parameters", {})
+        digit = params.get("digit", "")
+
+        if not digit:
+            logger.warning("DTMF message received without digit")
+            return None
+
+        logger.info(f"DTMF received: {digit}")
+
+        await self._call_event_handler("on_dtmf", message)
+
+        try:
+            return InputDTMFFrame(KeypadEntry(digit))
+        except ValueError:
+            # Invalid digit
+            logger.warning(f"Invalid DTMF digit: {digit}")
+            return None

src/pipecat/serializers/plivo.py

@@ -11,7 +11,6 @@ import json
 from typing import Optional
 
 from loguru import logger
-from pydantic import BaseModel
 
 from pipecat.audio.dtmf.types import KeypadEntry
 from pipecat.audio.utils import create_stream_resampler, pcm_to_ulaw, ulaw_to_pcm
@@ -42,13 +41,14 @@ class PlivoFrameSerializer(FrameSerializer):
     credentials to be provided.
     """
 
-    class InputParams(BaseModel):
+    class InputParams(FrameSerializer.InputParams):
         """Configuration parameters for PlivoFrameSerializer.
 
         Parameters:
             plivo_sample_rate: Sample rate used by Plivo, defaults to 8000 Hz.
             sample_rate: Optional override for pipeline input sample rate.
             auto_hang_up: Whether to automatically terminate call on EndFrame.
+            ignore_rtvi_messages: Inherited from base FrameSerializer, defaults to True.
         """
 
         plivo_sample_rate: int = 8000
@@ -72,11 +72,12 @@ class PlivoFrameSerializer(FrameSerializer):
             auth_token: Plivo auth token (required for auto hang-up).
             params: Configuration parameters.
         """
+        super().__init__(params or PlivoFrameSerializer.InputParams())
+
         self._stream_id = stream_id
         self._call_id = call_id
         self._auth_id = auth_id
         self._auth_token = auth_token
-        self._params = params or PlivoFrameSerializer.InputParams()
 
         self._plivo_sample_rate = self._params.plivo_sample_rate
         self._sample_rate = 0  # Pipeline input rate
@@ -140,6 +141,8 @@ class PlivoFrameSerializer(FrameSerializer):
 
             return json.dumps(answer)
         elif isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
+            if self.should_ignore_frame(frame):
+                return None
             return json.dumps(frame.message)
 
         # Return None for unhandled frames

src/pipecat/serializers/protobuf.py

@@ -8,6 +8,7 @@
 
 import dataclasses
 import json
+from typing import Optional
 
 from loguru import logger
 
@@ -60,9 +61,13 @@ class ProtobufFrameSerializer(FrameSerializer):
     }
     DESERIALIZABLE_FIELDS = {v: k for k, v in DESERIALIZABLE_TYPES.items()}
 
-    def __init__(self):
-        """Initialize the Protobuf frame serializer."""
-        pass
+    def __init__(self, params: Optional[FrameSerializer.InputParams] = None):
+        """Initialize the Protobuf frame serializer.
+
+        Args:
+            params: Configuration parameters.
+        """
+        super().__init__(params)
 
     async def serialize(self, frame: Frame) -> str | bytes | None:
         """Serialize a frame to Protocol Buffer binary format.
@@ -75,6 +80,8 @@ class ProtobufFrameSerializer(FrameSerializer):
         """
         # Wrapping this messages as a JSONFrame to send
         if isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
+            if self.should_ignore_frame(frame):
+                return None
             frame = MessageFrame(
                 data=json.dumps(frame.message),
             )
@@ -126,7 +133,7 @@ class ProtobufFrameSerializer(FrameSerializer):
         if "pts" in args_dict:
             del args_dict["pts"]
 
-        # Special handling for MessageFrame -> OutputTransportMessageUrgentFrame
+        # Special handling for MessageFrame -> InputTransportMessageFrame
         if class_name == MessageFrame:
             try:
                 msg = json.loads(args_dict["data"])

src/pipecat/serializers/telnyx.py

@@ -198,7 +198,7 @@ class TelnyxFrameSerializer(FrameSerializer):
                                     f"Telnyx call {call_control_id} was already terminated"
                                 )
                                 return
-                        except:
+                        except Exception:
                             pass  # Fall through to log the raw error
 
                         # Log other 422 errors

src/pipecat/serializers/twilio.py

@@ -11,7 +11,6 @@ import json
 from typing import Optional
 
 from loguru import logger
-from pydantic import BaseModel
 
 from pipecat.audio.dtmf.types import KeypadEntry
 from pipecat.audio.utils import create_stream_resampler, pcm_to_ulaw, ulaw_to_pcm
@@ -42,13 +41,14 @@ class TwilioFrameSerializer(FrameSerializer):
     credentials to be provided.
     """
 
-    class InputParams(BaseModel):
+    class InputParams(FrameSerializer.InputParams):
         """Configuration parameters for TwilioFrameSerializer.
 
         Parameters:
             twilio_sample_rate: Sample rate used by Twilio, defaults to 8000 Hz.
             sample_rate: Optional override for pipeline input sample rate.
             auto_hang_up: Whether to automatically terminate call on EndFrame.
+            ignore_rtvi_messages: Inherited from base FrameSerializer, defaults to True.
         """
 
         twilio_sample_rate: int = 8000
@@ -76,7 +76,7 @@ class TwilioFrameSerializer(FrameSerializer):
             edge: Twilio edge location (e.g., "sydney", "dublin"). Must be specified with region.
             params: Configuration parameters.
         """
-        self._params = params or TwilioFrameSerializer.InputParams()
+        super().__init__(params or TwilioFrameSerializer.InputParams())
 
         # Validate hangup-related parameters if auto_hang_up is enabled
         if self._params.auto_hang_up:
@@ -167,6 +167,8 @@ class TwilioFrameSerializer(FrameSerializer):
 
             return json.dumps(answer)
         elif isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
+            if self.should_ignore_frame(frame):
+                return None
             return json.dumps(frame.message)
 
         # Return None for unhandled frames
@@ -209,7 +211,7 @@ class TwilioFrameSerializer(FrameSerializer):
                             if error_data.get("code") == 20404:
                                 logger.debug(f"Twilio call {call_sid} was already terminated")
                                 return
-                        except:
+                        except Exception:
                             pass  # Fall through to log the raw error
 
                         # Log other 404 errors

src/pipecat/serializers/vonage.py

@@ -0,0 +1,183 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Vonage Audio Connector WebSocket serializer for Pipecat."""
+
+import json
+from typing import Optional
+
+from loguru import logger
+
+from pipecat.audio.dtmf.types import KeypadEntry
+from pipecat.audio.utils import create_stream_resampler
+from pipecat.frames.frames import (
+    AudioRawFrame,
+    Frame,
+    InputAudioRawFrame,
+    InputDTMFFrame,
+    InterruptionFrame,
+    OutputTransportMessageFrame,
+    OutputTransportMessageUrgentFrame,
+    StartFrame,
+)
+from pipecat.serializers.base_serializer import FrameSerializer
+
+
+class VonageFrameSerializer(FrameSerializer):
+    """Serializer for Vonage Video API Audio Connector WebSocket protocol.
+
+    This serializer converts between Pipecat frames and the Vonage Audio Connector
+    WebSocket streaming protocol.
+
+    Note:
+        Ref docs: https://developer.vonage.com/en/video/guides/audio-connector
+    """
+
+    class InputParams(FrameSerializer.InputParams):
+        """Configuration parameters for VonageFrameSerializer.
+
+        Parameters:
+            vonage_sample_rate: Sample rate used by Vonage, defaults to 16000 Hz.
+                Common values: 8000, 16000, 24000 Hz.
+            sample_rate: Optional override for pipeline input sample rate.
+            ignore_rtvi_messages: Inherited from base FrameSerializer, defaults to True.
+        """
+
+        vonage_sample_rate: int = 16000
+        sample_rate: Optional[int] = None
+
+    def __init__(self, params: Optional[InputParams] = None):
+        """Initialize the VonageFrameSerializer.
+
+        Args:
+            params: Configuration parameters.
+        """
+        super().__init__(params or VonageFrameSerializer.InputParams())
+
+        self._vonage_sample_rate = self._params.vonage_sample_rate
+        self._sample_rate = 0  # Pipeline input rate
+
+        self._input_resampler = create_stream_resampler()
+        self._output_resampler = create_stream_resampler()
+
+    async def setup(self, frame: StartFrame):
+        """Sets up the serializer with pipeline configuration.
+
+        Args:
+            frame: The StartFrame containing pipeline configuration.
+        """
+        self._sample_rate = self._params.sample_rate or frame.audio_in_sample_rate
+
+    async def serialize(self, frame: Frame) -> str | bytes | None:
+        """Serializes a Pipecat frame to Vonage WebSocket format.
+
+        Handles conversion of various frame types to Vonage WebSocket messages.
+
+        Args:
+            frame: The Pipecat frame to serialize.
+
+        Returns:
+            Serialized data as string (JSON commands) or bytes (audio), or None if the frame isn't handled.
+        """
+        if isinstance(frame, InterruptionFrame):
+            # Clear the audio buffer to stop playback immediately
+            answer = {"action": "clear"}
+            return json.dumps(answer)
+        elif isinstance(frame, AudioRawFrame):
+            data = frame.audio
+
+            # Output: Convert PCM at frame's rate to Vonage's sample rate (16-bit linear PCM)
+            serialized_data = await self._output_resampler.resample(
+                data, frame.sample_rate, self._vonage_sample_rate
+            )
+            if serialized_data is None or len(serialized_data) == 0:
+                # Ignoring in case we don't have audio
+                return None
+
+            # Vonage expects raw binary PCM data (not base64 encoded)
+            return serialized_data
+        elif isinstance(frame, (OutputTransportMessageFrame, OutputTransportMessageUrgentFrame)):
+            if self.should_ignore_frame(frame):
+                return None
+            # Allow sending custom JSON commands (e.g., notify)
+            return json.dumps(frame.message)
+
+        return None
+
+    async def deserialize(self, data: str | bytes) -> Frame | None:
+        """Deserializes Vonage WebSocket data to Pipecat frames.
+
+        Handles conversion of Vonage events to appropriate Pipecat frames.
+        - Binary messages contain audio data (16-bit linear PCM)
+        - Text messages contain JSON events (websocket:connected, websocket:cleared, dtmf, etc.)
+
+        Args:
+            data: The raw WebSocket data from Vonage.
+
+        Returns:
+            A Pipecat frame corresponding to the Vonage event, or None if unhandled.
+        """
+        # Check if this is binary audio data
+        if isinstance(data, bytes):
+            # Binary message = audio data (16-bit linear PCM)
+            payload = data
+
+            # Input: Convert Vonage's PCM audio to pipeline sample rate
+            deserialized_data = await self._input_resampler.resample(
+                payload,
+                self._vonage_sample_rate,
+                self._sample_rate,
+            )
+            if deserialized_data is None or len(deserialized_data) == 0:
+                # Ignoring in case we don't have audio
+                return None
+
+            audio_frame = InputAudioRawFrame(
+                audio=deserialized_data,
+                num_channels=1,  # Vonage uses mono audio
+                sample_rate=self._sample_rate,  # Use the configured pipeline input rate
+            )
+            return audio_frame
+        else:
+            # Text message = JSON event
+            try:
+                message = json.loads(data)
+                event = message.get("event")
+
+                # Handle different event types
+                if event == "websocket:connected":
+                    logger.debug(
+                        f"Vonage WebSocket connected: content-type={message.get('content-type')}"
+                    )
+                    return None
+                elif event == "websocket:cleared":
+                    logger.debug("Vonage audio buffer cleared")
+                    return None
+                elif event == "websocket:notify":
+                    logger.debug(f"Vonage notify event: {message.get('payload')}")
+                    return None
+                elif event == "websocket:dtmf":
+                    # Handle DTMF input
+                    # Vonage may send digit in different formats, try both
+                    digit = message.get("digit") or message.get("dtmf", {}).get("digit")
+                    if digit is None:
+                        logger.warning(f"DTMF event received but no digit found: {message}")
+                        return None
+
+                    digit = str(digit)
+                    logger.debug(f"Received DTMF digit: {digit}")
+                    try:
+                        return InputDTMFFrame(KeypadEntry(digit))
+                    except ValueError:
+                        logger.warning(f"Invalid DTMF digit received: {digit}")
+                        return None
+                else:
+                    logger.debug(f"Vonage event: {event}")
+                    return None
+
+            except json.JSONDecodeError:
+                logger.warning(f"Failed to parse JSON message from Vonage: {data}")
+                return None

src/pipecat/services/ai_service.py

@@ -10,7 +10,8 @@ Provides the foundation for all AI services in the Pipecat framework, including
 model management, settings handling, and frame processing lifecycle methods.
 """
 
-from typing import Any, AsyncGenerator, Dict, Mapping
+import warnings
+from typing import Any, AsyncGenerator, Dict
 
 from loguru import logger
 
@@ -23,6 +24,7 @@ from pipecat.frames.frames import (
 )
 from pipecat.metrics.metrics import MetricsData
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.services.settings import ServiceSettings
 
 
 class AIService(FrameProcessor):
@@ -34,34 +36,38 @@ class AIService(FrameProcessor):
     this base infrastructure.
     """
 
-    def __init__(self, **kwargs):
+    def __init__(self, settings: ServiceSettings | None = None, **kwargs):
         """Initialize the AI service.
 
         Args:
+            settings: The runtime-updatable settings for the AI service.
             **kwargs: Additional arguments passed to the parent FrameProcessor.
         """
         super().__init__(**kwargs)
-        self._model_name: str = ""
-        self._settings: Dict[str, Any] = {}
+        self._settings: ServiceSettings = (
+            settings
+            # Here in case subclass doesn't implement more specific settings
+            # (which hopefully should be rare)
+            or ServiceSettings()
+        )
+        self._sync_model_name_to_metrics()
         self._session_properties: Dict[str, Any] = {}
+        self._tracing_enabled: bool = False
+        self._tracing_context = None
 
-    @property
-    def model_name(self) -> str:
-        """Get the current model name.
+    def _sync_model_name_to_metrics(self):
+        """Sync the current AI model name (in `self._settings.model`) for usage in metrics.
 
-        Returns:
-            The name of the AI model being used.
-        """
-        return self._model_name
-
-    def set_model_name(self, model: str):
-        """Set the AI model name and update metrics.
+        We don't store model name here because there's already a single source
+        of truth for it in `self._settings.model`. This method is just for
+        syncing the model name to the metrics data.
 
         Args:
             model: The name of the AI model to use.
         """
-        self._model_name = model
-        self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
+        self.set_core_metrics_data(
+            MetricsData(processor=self.name, model=self._settings.model or "")
+        )
 
     async def start(self, frame: StartFrame):
         """Start the AI service.
@@ -72,7 +78,9 @@ class AIService(FrameProcessor):
         Args:
             frame: The start frame containing initialization parameters.
         """
-        pass
+        self._settings.validate_complete()
+        self._tracing_enabled = frame.enable_tracing
+        self._tracing_context = frame.tracing_context
 
     async def stop(self, frame: EndFrame):
         """Stop the AI service.
@@ -96,44 +104,82 @@ class AIService(FrameProcessor):
         """
         pass
 
-    async def _update_settings(self, settings: Mapping[str, Any]):
-        from pipecat.services.openai.realtime.events import SessionProperties
+    async def _update_settings(self, delta: ServiceSettings) -> Dict[str, Any]:
+        """Apply a settings delta and return the changed fields.
 
-        for key, value in settings.items():
-            logger.debug("Update request for:", key, value)
+        The delta is applied to ``_settings`` and a dict mapping each changed
+        field name to its **pre-update** value is returned.  The ``model``
+        field is handled specially: when it changes, ``set_model_name`` is
+        called.
 
-            if key in self._settings:
-                logger.info(f"Updating LLM setting {key} to: [{value}]")
-                self._settings[key] = value
-            elif key in SessionProperties.model_fields:
-                logger.debug("Attempting to update", key, value)
+        Concrete services should override this method (calling ``super()``)
+        to react to specific changed fields (e.g. reconnect on voice change).
 
-                try:
-                    from pipecat.services.openai.realtime.events import TurnDetection
+        Args:
+            delta: A delta-mode settings object.
 
-                    if isinstance(self._session_properties, SessionProperties):
-                        current_properties = self._session_properties
-                    else:
-                        current_properties = SessionProperties(**self._session_properties)
+        Returns:
+            Dict mapping changed field names to their previous values.
+        """
+        changed = self._settings.apply_update(delta)
 
-                    if key == "turn_detection" and isinstance(value, dict):
-                        turn_detection = TurnDetection(**value)
-                        setattr(current_properties, key, turn_detection)
-                    else:
-                        setattr(current_properties, key, value)
+        if "model" in changed:
+            self._sync_model_name_to_metrics()
 
-                    validated_properties = SessionProperties.model_validate(
-                        current_properties.model_dump()
-                    )
-                    logger.info(f"Updating LLM setting {key} to: [{value}]")
-                    self._session_properties = validated_properties.model_dump()
-                except Exception as e:
-                    logger.warning(f"Unexpected error updating session property {key}: {e}")
-            elif key == "model":
-                logger.info(f"Updating LLM setting {key} to: [{value}]")
-                self.set_model_name(value)
-            else:
-                logger.warning(f"Unknown setting for {self.name} service: {key}")
+        if changed:
+            logger.info(f"{self.name}: updated settings fields: {set(changed)}")
+
+        return changed
+
+    def _warn_init_param_moved_to_settings(
+        self,
+        param_name: str,
+        settings_field: str | None = None,
+        stacklevel: int = 3,
+    ):
+        """Warn that an ``__init__`` param has moved to ``Settings``.
+
+        Emits a ``DeprecationWarning`` directing users to the canonical
+        ``settings=ServiceClass.Settings(field=...)`` API.
+
+        Args:
+            param_name: Name of the deprecated ``__init__`` parameter.
+            settings_field: The corresponding field on the ``Settings``
+                dataclass, if different from *param_name*.  When ``None``
+                the message omits the field hint.
+            stacklevel: Stack depth for the warning.  Default ``3`` targets
+                the caller's caller (i.e. user code that instantiated the
+                service).
+        """
+        label = f"{type(self).__name__}.Settings"
+        if settings_field:
+            msg = (
+                f"The `{param_name}` parameter is deprecated. "
+                f"Use `settings={label}({settings_field}=...)` instead. "
+                f"If both are provided, `settings` takes precedence."
+            )
+        else:
+            msg = (
+                f"The `{param_name}` parameter is deprecated. "
+                f"Use `settings={label}(...)` instead. "
+                f"If both are provided, `settings` takes precedence."
+            )
+        with warnings.catch_warnings():
+            warnings.simplefilter("always")
+            warnings.warn(msg, DeprecationWarning, stacklevel=stacklevel)
+
+    def _warn_unhandled_updated_settings(self, unhandled):
+        """Log a warning for settings changes that won't take effect at runtime.
+
+        Convenience helper for ``_update_settings`` overrides.  Accepts any
+        iterable of field names (a ``dict``, ``set``, ``dict_keys``, etc.).
+
+        Args:
+            unhandled: Field names that changed but are not applied.
+        """
+        if unhandled:
+            fields = ", ".join(sorted(unhandled))
+            logger.warning(f"{self.name}: runtime update of [{fields}] is not currently supported")
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         """Process frames and handle service lifecycle.
@@ -148,11 +194,11 @@ class AIService(FrameProcessor):
         await super().process_frame(frame, direction)
 
         if isinstance(frame, StartFrame):
-            await self.start(frame)
-        elif isinstance(frame, CancelFrame):
-            await self.cancel(frame)
+            await self._start(frame)
         elif isinstance(frame, EndFrame):
-            await self.stop(frame)
+            await self._stop(frame)
+        elif isinstance(frame, CancelFrame):
+            await self._cancel(frame)
 
     async def process_generator(self, generator: AsyncGenerator[Frame | None, None]):
         """Process frames from an async generator.
@@ -169,3 +215,21 @@ class AIService(FrameProcessor):
                     await self.push_error_frame(f)
                 else:
                     await self.push_frame(f)
+
+    async def _start(self, frame: StartFrame):
+        try:
+            await self.start(frame)
+        except Exception as e:
+            logger.error(f"{self}: exception processing {frame}: {e}")
+
+    async def _stop(self, frame: EndFrame):
+        try:
+            await self.stop(frame)
+        except Exception as e:
+            logger.error(f"{self}: exception processing {frame}: {e}")
+
+    async def _cancel(self, frame: CancelFrame):
+        try:
+            await self.cancel(frame)
+        except Exception as e:
+            logger.error(f"{self}: exception processing {frame}: {e}")

src/pipecat/services/anthropic/llm.py

@@ -16,7 +16,7 @@ import copy
 import io
 import json
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any, Dict, List, Literal, Optional, Union
 
 import httpx
@@ -38,11 +38,9 @@ from pipecat.frames.frames import (
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesFrame,
-    LLMTextFrame,
     LLMThoughtEndFrame,
     LLMThoughtStartFrame,
     LLMThoughtTextFrame,
-    LLMUpdateSettingsFrame,
     UserImageRawFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
@@ -59,6 +57,8 @@ from pipecat.processors.aggregators.openai_llm_context import (
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.llm_service import FunctionCallFromLLM, LLMService
+from pipecat.services.settings import NOT_GIVEN as _NOT_GIVEN
+from pipecat.services.settings import LLMSettings, _NotGiven, is_given
 from pipecat.utils.tracing.service_decorators import traced_llm
 
 try:
@@ -69,6 +69,52 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+class AnthropicThinkingConfig(BaseModel):
+    """Configuration for extended thinking.
+
+    Parameters:
+        type: Type of thinking mode (currently only "enabled" or "disabled").
+        budget_tokens: Maximum number of tokens for thinking.
+            With today's models, the minimum is 1024.
+            Currently required when type is "enabled", not allowed when "disabled".
+    """
+
+    # Why `| str` here? To not break compatibility in case Anthropic adds
+    # more types in the future.
+    type: Literal["enabled", "disabled"] | str
+
+    # No client-side validation on budget_tokens — we let the server
+    # enforce the rules so we stay forward-compatible if they change.
+    budget_tokens: Optional[int] = None
+
+
+@dataclass
+class AnthropicLLMSettings(LLMSettings):
+    """Settings for AnthropicLLMService.
+
+    Parameters:
+        enable_prompt_caching: Whether to enable prompt caching.
+        thinking: Extended thinking configuration.
+    """
+
+    enable_prompt_caching: bool | _NotGiven = field(default_factory=lambda: _NOT_GIVEN)
+    thinking: Union["AnthropicLLMService.ThinkingConfig", _NotGiven] = field(
+        default_factory=lambda: _NOT_GIVEN
+    )
+
+    @classmethod
+    def from_mapping(cls, settings):
+        """Convert a plain dict to settings, coercing thinking dicts.
+
+        For backward compatibility, a ``thinking`` value that is a plain dict
+        is converted to a :class:`AnthropicLLMService.ThinkingConfig`.
+        """
+        instance = super().from_mapping(settings)
+        if is_given(instance.thinking) and isinstance(instance.thinking, dict):
+            instance.thinking = AnthropicLLMService.ThinkingConfig(**instance.thinking)
+        return instance
+
+
 @dataclass
 class AnthropicContextAggregatorPair:
     """Pair of context aggregators for Anthropic conversations.
@@ -115,30 +161,22 @@ class AnthropicLLMService(LLMService):
     Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex.
     """
 
+    Settings = AnthropicLLMSettings
+    _settings: Settings
+
     # Overriding the default adapter to use the Anthropic one.
     adapter_class = AnthropicLLMAdapter
 
-    class ThinkingConfig(BaseModel):
-        """Configuration for extended thinking.
-
-        Parameters:
-            type: Type of thinking mode (currently only "enabled" or "disabled").
-            budget_tokens: Maximum number of tokens for thinking.
-                With today's models, the minimum is 1024.
-                Only allowed if type is "enabled".
-        """
-
-        # Why `| str` here? To not break compatibility in case Anthropic adds
-        # more types in the future.
-        type: Literal["enabled", "disabled"] | str
-
-        # Why not enforce minimnum of 1024 here? To not break compatibility in
-        # case Anthropic changes this requirement in the future.
-        budget_tokens: int
+    # Backward compatibility: ThinkingConfig used to be defined inline here.
+    ThinkingConfig = AnthropicThinkingConfig
 
     class InputParams(BaseModel):
         """Input parameters for Anthropic model inference.
 
+        .. deprecated:: 0.0.105
+            Use ``AnthropicLLMService.Settings`` instead. Pass settings directly via the
+            ``settings`` parameter of :class:`AnthropicLLMService`.
+
         Parameters:
             enable_prompt_caching: Whether to enable the prompt caching feature.
             enable_prompt_caching_beta (deprecated): Whether to enable the beta prompt caching feature.
@@ -184,8 +222,9 @@ class AnthropicLLMService(LLMService):
         self,
         *,
         api_key: str,
-        model: str = "claude-sonnet-4-5-20250929",
+        model: Optional[str] = None,
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
         client=None,
         retry_timeout_secs: Optional[float] = 5.0,
         retry_on_timeout: Optional[bool] = False,
@@ -195,38 +234,87 @@ class AnthropicLLMService(LLMService):
 
         Args:
             api_key: Anthropic API key for authentication.
-            model: Model name to use. Defaults to "claude-sonnet-4-5-20250929".
+            model: Model name to use.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AnthropicLLMService.Settings(model=...)`` instead.
+
             params: Optional model parameters for inference.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AnthropicLLMService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings for this service.  When both
+                deprecated parameters and *settings* are provided, *settings*
+                values take precedence.
             client: Optional custom Anthropic client instance.
             retry_timeout_secs: Request timeout in seconds for retry logic.
             retry_on_timeout: Whether to retry the request once if it times out.
             **kwargs: Additional arguments passed to parent LLMService.
         """
-        super().__init__(**kwargs)
-        params = params or AnthropicLLMService.InputParams()
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="claude-sonnet-4-6",
+            system_instruction=None,
+            max_tokens=4096,
+            enable_prompt_caching=False,
+            temperature=NOT_GIVEN,
+            top_k=NOT_GIVEN,
+            top_p=NOT_GIVEN,
+            frequency_penalty=None,
+            presence_penalty=None,
+            seed=None,
+            filter_incomplete_user_turns=False,
+            user_turn_completion_config=None,
+            thinking=NOT_GIVEN,
+            extra={},
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.max_tokens = params.max_tokens
+                default_settings.temperature = params.temperature
+                default_settings.top_k = params.top_k
+                default_settings.top_p = params.top_p
+                default_settings.thinking = params.thinking
+                if isinstance(params.extra, dict):
+                    default_settings.extra = params.extra
+                # Handle enable_prompt_caching / enable_prompt_caching_beta
+                enable_prompt_caching = params.enable_prompt_caching
+                if params.enable_prompt_caching_beta is not None:
+                    import warnings
+
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("always")
+                        warnings.warn(
+                            "enable_prompt_caching_beta is deprecated. "
+                            "Use enable_prompt_caching instead.",
+                            DeprecationWarning,
+                            stacklevel=2,
+                        )
+                    if enable_prompt_caching is None:
+                        enable_prompt_caching = params.enable_prompt_caching_beta
+                default_settings.enable_prompt_caching = enable_prompt_caching or False
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(settings=default_settings, **kwargs)
         self._client = client or AsyncAnthropic(
             api_key=api_key
         )  # if the client is provided, use it and remove it, otherwise create a new one
-        self.set_model_name(model)
         self._retry_timeout_secs = retry_timeout_secs
         self._retry_on_timeout = retry_on_timeout
-        self._settings = {
-            "max_tokens": params.max_tokens,
-            "enable_prompt_caching": (
-                params.enable_prompt_caching
-                if params.enable_prompt_caching is not None
-                else (
-                    params.enable_prompt_caching_beta
-                    if params.enable_prompt_caching_beta is not None
-                    else False
-                )
-            ),
-            "temperature": params.temperature,
-            "top_k": params.top_k,
-            "top_p": params.top_p,
-            "thinking": params.thinking,
-            "extra": params.extra if isinstance(params.extra, dict) else {},
-        }
+        if self._settings.system_instruction:
+            logger.debug(f"{self}: Using system instruction: {self._settings.system_instruction}")
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate usage metrics.
@@ -261,11 +349,20 @@ class AnthropicLLMService(LLMService):
             response = await api_call(**params)
             return response
 
-    async def run_inference(self, context: LLMContext | OpenAILLMContext) -> Optional[str]:
+    async def run_inference(
+        self,
+        context: LLMContext | OpenAILLMContext,
+        max_tokens: Optional[int] = None,
+        system_instruction: Optional[str] = None,
+    ) -> Optional[str]:
         """Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context.
 
         Args:
             context: The LLM context containing conversation history.
+            max_tokens: Optional maximum number of tokens to generate. If provided,
+                overrides the service's default max_tokens setting.
+            system_instruction: Optional system instruction to use for this inference.
+                If provided, overrides any system instruction in the context.
 
         Returns:
             The LLM's response as a string, or None if no response is generated.
@@ -276,7 +373,7 @@ class AnthropicLLMService(LLMService):
         if isinstance(context, LLMContext):
             adapter: AnthropicLLMAdapter = self.get_llm_adapter()
             invocation_params = adapter.get_llm_invocation_params(
-                context, enable_prompt_caching=self._settings["enable_prompt_caching"]
+                context, enable_prompt_caching=self._settings.enable_prompt_caching
             )
             messages = invocation_params["messages"]
             system = invocation_params["system"]
@@ -287,23 +384,32 @@ class AnthropicLLMService(LLMService):
             system = getattr(context, "system", NOT_GIVEN)
             tools = context.tools or []
 
+        # Override system instruction if provided
+        if system_instruction is not None:
+            if system and system is not NOT_GIVEN:
+                logger.warning(
+                    f"{self}: Both system_instruction and a system message in context are set."
+                    " Using system_instruction."
+                )
+            system = system_instruction
+
         # Build params using the same method as streaming completions
         params = {
-            "model": self.model_name,
-            "max_tokens": self._settings["max_tokens"],
+            "model": self._settings.model,
+            "max_tokens": max_tokens if max_tokens is not None else self._settings.max_tokens,
             "stream": False,
-            "temperature": self._settings["temperature"],
-            "top_k": self._settings["top_k"],
-            "top_p": self._settings["top_p"],
+            "temperature": self._settings.temperature,
+            "top_k": self._settings.top_k,
+            "top_p": self._settings.top_p,
             "messages": messages,
             "system": system,
             "tools": tools,
             "betas": ["interleaved-thinking-2025-05-14"],
         }
-        if self._settings["thinking"]:
-            params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
+        if self._settings.thinking:
+            params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
 
-        params.update(self._settings["extra"])
+        params.update(self._settings.extra)
 
         # LLM completion
         response = await self._client.beta.messages.create(**params)
@@ -353,15 +459,22 @@ class AnthropicLLMService(LLMService):
         # Universal LLMContext
         if isinstance(context, LLMContext):
             adapter: AnthropicLLMAdapter = self.get_llm_adapter()
-            params = adapter.get_llm_invocation_params(
-                context, enable_prompt_caching=self._settings["enable_prompt_caching"]
+            params: AnthropicLLMInvocationParams = adapter.get_llm_invocation_params(
+                context, enable_prompt_caching=self._settings.enable_prompt_caching
             )
+            if self._settings.system_instruction:
+                if params["system"] is not NOT_GIVEN:
+                    logger.warning(
+                        f"{self}: Both system_instruction and a system message in context are"
+                        " set. Using system_instruction."
+                    )
+                params["system"] = self._settings.system_instruction
             return params
 
         # Anthropic-specific context
         messages = (
             context.get_messages_with_cache_control_markers()
-            if self._settings["enable_prompt_caching"]
+            if self._settings.enable_prompt_caching
             else context.messages
         )
         return AnthropicLLMInvocationParams(
@@ -403,22 +516,22 @@ class AnthropicLLMService(LLMService):
             await self.start_ttfb_metrics()
 
             params = {
-                "model": self.model_name,
-                "max_tokens": self._settings["max_tokens"],
+                "model": self._settings.model,
+                "max_tokens": self._settings.max_tokens,
                 "stream": True,
-                "temperature": self._settings["temperature"],
-                "top_k": self._settings["top_k"],
-                "top_p": self._settings["top_p"],
+                "temperature": self._settings.temperature,
+                "top_k": self._settings.top_k,
+                "top_p": self._settings.top_p,
             }
 
             # Add thinking parameter if set
-            if self._settings["thinking"]:
-                params["thinking"] = self._settings["thinking"].model_dump(exclude_unset=True)
+            if self._settings.thinking:
+                params["thinking"] = self._settings.thinking.model_dump(exclude_unset=True)
 
             # Messages, system, tools
             params.update(params_from_context)
 
-            params.update(self._settings["extra"])
+            params.update(self._settings.extra)
 
             # "Interleaved thinking" needed to allow thinking between sequences
             # of function calls, when extended thinking is enabled.
@@ -439,7 +552,7 @@ class AnthropicLLMService(LLMService):
 
                 if event.type == "content_block_delta":
                     if hasattr(event.delta, "text"):
-                        await self.push_frame(LLMTextFrame(event.delta.text))
+                        await self._push_llm_text(event.delta.text)
                         completion_tokens_estimate += self._estimate_tokens(event.delta.text)
                     elif hasattr(event.delta, "partial_json") and tool_use_block:
                         json_accumulator += event.delta.partial_json
@@ -572,11 +685,9 @@ class AnthropicLLMService(LLMService):
             # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
             # LLMContext with it
             context = AnthropicLLMContext.from_messages(frame.messages)
-        elif isinstance(frame, LLMUpdateSettingsFrame):
-            await self._update_settings(frame.settings)
         elif isinstance(frame, LLMEnablePromptCachingFrame):
             logger.debug(f"Setting enable prompt caching to: [{frame.enable}]")
-            self._settings["enable_prompt_caching"] = frame.enable
+            self._settings.enable_prompt_caching = frame.enable
         else:
             await self.push_frame(frame, direction)
 
@@ -1135,7 +1246,7 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator):
             frame: Frame containing function call result.
         """
         if frame.result:
-            result = json.dumps(frame.result)
+            result = json.dumps(frame.result, ensure_ascii=False)
             await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
         else:
             await self._update_function_call_result(

src/pipecat/services/assemblyai/models.py

@@ -12,7 +12,8 @@ transcription WebSocket messages and connection configuration.
 
 from typing import List, Literal, Optional
 
-from pydantic import BaseModel, Field
+from loguru import logger
+from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 
 class Word(BaseModel):
@@ -68,8 +69,16 @@ class TurnMessage(BaseMessage):
         transcript: The transcribed text for this turn.
         end_of_turn_confidence: Confidence score for end-of-turn detection.
         words: List of individual words with timing and confidence data.
+        language_code: Detected language code (e.g., "es", "fr"). Only present with
+            complete utterances or when end_of_turn is True.
+        language_confidence: Confidence score (0-1) for language detection. Only present
+            with complete utterances or when end_of_turn is True.
+        speaker: Speaker label (e.g., "A", "B"). Only present when speaker_labels is
+            enabled and end_of_turn is True. Maps to 'speaker_label' in JSON response.
     """
 
+    model_config = ConfigDict(populate_by_name=True)
+
     type: Literal["Turn"] = "Turn"
     turn_order: int
     turn_is_formatted: bool
@@ -77,6 +86,21 @@ class TurnMessage(BaseMessage):
     transcript: str
     end_of_turn_confidence: float
     words: List[Word]
+    language_code: Optional[str] = None
+    language_confidence: Optional[float] = None
+    speaker: Optional[str] = Field(default=None, alias="speaker_label")
+
+
+class SpeechStartedMessage(BaseMessage):
+    """Message sent when speech is first detected in the audio stream.
+
+    Parameters:
+        type: Always "SpeechStarted" for this message type.
+        timestamp: Audio timestamp in milliseconds when speech was detected.
+    """
+
+    type: Literal["SpeechStarted"] = "SpeechStarted"
+    timestamp: int
 
 
 class TerminationMessage(BaseMessage):
@@ -94,32 +118,69 @@ class TerminationMessage(BaseMessage):
 
 
 # Union type for all possible message types
-AnyMessage = BeginMessage | TurnMessage | TerminationMessage
+AnyMessage = BeginMessage | TurnMessage | SpeechStartedMessage | TerminationMessage
 
 
 class AssemblyAIConnectionParams(BaseModel):
     """Configuration parameters for AssemblyAI WebSocket connection.
 
+    .. deprecated:: 0.0.105
+        Use ``settings=AssemblyAISTTService.Settings(foo=...)`` instead.
+
     Parameters:
         sample_rate: Audio sample rate in Hz. Defaults to 16000.
         encoding: Audio encoding format. Defaults to "pcm_s16le".
-        formatted_finals: Whether to enable transcript formatting. Defaults to True.
-        word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds.
         end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection.
-        min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn.
+        min_turn_silence: Minimum silence duration when confident about end-of-turn.
+        min_end_of_turn_silence_when_confident: DEPRECATED. Use min_turn_silence instead.
         max_turn_silence: Maximum silence duration before forcing end-of-turn.
         keyterms_prompt: List of key terms to guide transcription. Will be JSON serialized before sending.
-        speech_model: Select between English and multilingual models. Defaults to "universal-streaming-english".
+        prompt: Optional text prompt to guide the transcription. Only used when speech_model is "u3-rt-pro".
+        speech_model: Select between English, multilingual, and u3-rt-pro models. Defaults to "u3-rt-pro".
+        language_detection: Enable automatic language detection. Only applicable to
+            universal-streaming-multilingual. When enabled, Turn messages include
+            language_code and language_confidence fields. Defaults to None (not sent).
+        format_turns: Whether to format transcript turns. Only applicable to
+            universal-streaming-english and universal-streaming-multilingual models.
+            For u3-rt-pro, formatting is automatic and built-in. Defaults to True.
+        speaker_labels: Enable speaker diarization. When enabled, final transcripts
+            (end_of_turn=True) include a speaker field identifying the speaker
+            (e.g., "Speaker A", "Speaker B"). Defaults to None (not sent).
+        vad_threshold: Voice activity detection confidence threshold. Only applicable to
+            u3-rt-pro. The confidence threshold (0.0 to 1.0) for classifying audio frames
+            as silence. Frames with VAD confidence below this value are considered silent.
+            Increase for noisy environments to reduce false speech detection. Defaults to
+            0.3 (API default). For best performance when using with external VAD (e.g., Silero),
+            align this value with your VAD's activation threshold to avoid the "dead zone"
+            where AssemblyAI transcribes speech that your VAD hasn't detected yet.
+            Defaults to None (not sent).
     """
 
     sample_rate: int = 16000
     encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le"
-    formatted_finals: bool = True
-    word_finalization_max_wait_time: Optional[int] = None
     end_of_turn_confidence_threshold: Optional[float] = None
-    min_end_of_turn_silence_when_confident: Optional[int] = None
+    min_turn_silence: Optional[int] = None
+    min_end_of_turn_silence_when_confident: Optional[int] = None  # Deprecated
     max_turn_silence: Optional[int] = None
     keyterms_prompt: Optional[List[str]] = None
-    speech_model: Literal["universal-streaming-english", "universal-streaming-multilingual"] = (
-        "universal-streaming-english"
-    )
+    prompt: Optional[str] = None
+    speech_model: Literal[
+        "universal-streaming-english", "universal-streaming-multilingual", "u3-rt-pro"
+    ] = "u3-rt-pro"
+    language_detection: Optional[bool] = None
+    format_turns: bool = True
+    speaker_labels: Optional[bool] = None
+    vad_threshold: Optional[float] = None
+
+    @model_validator(mode="after")
+    def handle_deprecated_param(self):
+        """Handle deprecated min_end_of_turn_silence_when_confident parameter."""
+        if self.min_end_of_turn_silence_when_confident is not None:
+            logger.warning(
+                "The 'min_end_of_turn_silence_when_confident' parameter is deprecated and will be "
+                "removed in a future version. Please use 'min_turn_silence' instead."
+            )
+            # If min_turn_silence is not set, use the deprecated value
+            if self.min_turn_silence is None:
+                self.min_turn_silence = self.min_end_of_turn_silence_when_confident
+        return self

src/pipecat/services/assemblyai/stt.py

@@ -12,7 +12,8 @@ WebSocket API for streaming audio transcription.
 
 import asyncio
 import json
-from typing import Any, AsyncGenerator, Dict
+from dataclasses import dataclass, field
+from typing import Any, AsyncGenerator, Dict, List, Optional
 from urllib.parse import urlencode
 
 from loguru import logger
@@ -25,10 +26,14 @@ from pipecat.frames.frames import (
     InterimTranscriptionFrame,
     StartFrame,
     TranscriptionFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
     VADUserStartedSpeakingFrame,
     VADUserStoppedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
+from pipecat.services.stt_latency import ASSEMBLYAI_TTFS_P99
 from pipecat.services.stt_service import WebsocketSTTService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
@@ -38,6 +43,7 @@ from .models import (
     AssemblyAIConnectionParams,
     BaseMessage,
     BeginMessage,
+    SpeechStartedMessage,
     TerminationMessage,
     TurnMessage,
 )
@@ -51,6 +57,69 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+def map_language_from_assemblyai(language_code: str) -> Language:
+    """Map AssemblyAI language codes to Pipecat Language enum.
+
+    AssemblyAI returns simple language codes like "es", "fr", etc.
+    This function maps them to the corresponding Language enum values.
+
+    Args:
+        language_code: AssemblyAI language code (e.g., "es", "fr", "de")
+
+    Returns:
+        Corresponding Language enum value, defaulting to Language.EN if not found.
+    """
+    try:
+        # Try to match the language code directly
+        return Language(language_code.lower())
+    except ValueError:
+        logger.warning(
+            f"Unknown language code from AssemblyAI: {language_code}, defaulting to English"
+        )
+        return Language.EN
+
+
+@dataclass
+class AssemblyAISTTSettings(STTSettings):
+    """Settings for AssemblyAISTTService.
+
+    Parameters:
+        formatted_finals: Whether to enable transcript formatting.
+        word_finalization_max_wait_time: Maximum time to wait for word
+            finalization in milliseconds.
+        end_of_turn_confidence_threshold: Confidence threshold for
+            end-of-turn detection.
+        min_turn_silence: Minimum silence duration when confident about
+            end-of-turn.
+        max_turn_silence: Maximum silence duration before forcing
+            end-of-turn.
+        keyterms_prompt: List of key terms to guide transcription.
+        prompt: Optional text prompt to guide the transcription. Only
+            used when model is "u3-rt-pro".
+        language_detection: Enable automatic language detection.
+        format_turns: Whether to format transcript turns.
+        speaker_labels: Enable speaker diarization.
+        vad_threshold: VAD confidence threshold (0.0–1.0) for classifying
+            audio frames as silence. Only applicable to u3-rt-pro.
+    """
+
+    formatted_finals: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    word_finalization_max_wait_time: int | None | _NotGiven = field(
+        default_factory=lambda: NOT_GIVEN
+    )
+    end_of_turn_confidence_threshold: float | None | _NotGiven = field(
+        default_factory=lambda: NOT_GIVEN
+    )
+    min_turn_silence: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    max_turn_silence: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    keyterms_prompt: List[str] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    prompt: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    language_detection: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    format_turns: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    speaker_labels: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
 class AssemblyAISTTService(WebsocketSTTService):
     """AssemblyAI real-time speech-to-text service.
 
@@ -59,14 +128,23 @@ class AssemblyAISTTService(WebsocketSTTService):
     for audio processing and connection management.
     """
 
+    Settings = AssemblyAISTTSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
         api_key: str,
-        language: Language = Language.EN,  # AssemblyAI only supports English
+        language: Optional[Language] = None,
         api_endpoint_base_url: str = "wss://streaming.assemblyai.com/v3/ws",
-        connection_params: AssemblyAIConnectionParams = AssemblyAIConnectionParams(),
+        sample_rate: int = 16000,
+        encoding: str = "pcm_s16le",
+        connection_params: Optional[AssemblyAIConnectionParams] = None,
         vad_force_turn_endpoint: bool = True,
+        should_interrupt: bool = True,
+        speaker_format: Optional[str] = None,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = ASSEMBLYAI_TTFS_P99,
         **kwargs,
     ):
         """Initialize the AssemblyAI STT service.
@@ -74,18 +152,140 @@ class AssemblyAISTTService(WebsocketSTTService):
         Args:
             api_key: AssemblyAI API key for authentication.
             language: Language code for transcription. Defaults to English (Language.EN).
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AssemblyAISTTService.Settings(language=...)`` instead.
+
             api_endpoint_base_url: WebSocket endpoint URL. Defaults to AssemblyAI's streaming endpoint.
-            connection_params: Connection configuration parameters. Defaults to AssemblyAIConnectionParams().
-            vad_force_turn_endpoint: Whether to force turn endpoint on VAD stop. Defaults to True.
+            sample_rate: Audio sample rate in Hz. Defaults to 16000.
+            encoding: Audio encoding format. Defaults to "pcm_s16le".
+            connection_params: Connection configuration parameters.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AssemblyAISTTService.Settings(...)`` instead.
+
+            vad_force_turn_endpoint: Controls turn detection mode.
+                When True (Pipecat mode, default): Forces AssemblyAI to return finals ASAP
+                so Pipecat's turn detection (e.g., Smart Turn) decides when the user is done.
+                - min_turn_silence defaults to 100ms (user can override)
+                - max_turn_silence is ALWAYS set equal to min_turn_silence
+                - VAD stop sends ForceEndpoint as ceiling
+                - No UserStarted/StoppedSpeakingFrame emitted from STT
+                When False (AssemblyAI turn detection mode, u3-rt-pro only): AssemblyAI's model
+                controls turn endings using built-in turn detection.
+                - Uses AssemblyAI API defaults for all parameters (unless user explicitly sets them)
+                - Emits UserStarted/StoppedSpeakingFrame from STT
+                - No ForceEndpoint on VAD stop
+            should_interrupt: Whether to interrupt the bot when the user starts speaking
+                in AssemblyAI turn detection mode (vad_force_turn_endpoint=False). Only applies
+                when using AssemblyAI's built-in turn detection. Defaults to True.
+            speaker_format: Optional format string for speaker labels when diarization is enabled.
+                Use {speaker} for speaker label and {text} for transcript text.
+                Example: "<{speaker}>{text}</{speaker}>" or "{speaker}: {text}"
+                If None, transcript text is not modified. Defaults to None.
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
             **kwargs: Additional arguments passed to parent STTService class.
         """
-        super().__init__(sample_rate=connection_params.sample_rate, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="u3-rt-pro",
+            language=Language.EN,
+            formatted_finals=True,
+            word_finalization_max_wait_time=None,
+            end_of_turn_confidence_threshold=None,
+            min_turn_silence=None,
+            max_turn_silence=None,
+            keyterms_prompt=None,
+            prompt=None,
+            language_detection=None,
+            format_turns=True,
+            speaker_labels=None,
+            vad_threshold=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if language is not None:
+            self._warn_init_param_moved_to_settings("language", "language")
+            default_settings.language = language
+
+        # 3. Apply connection_params overrides (deprecated) — only if settings not provided
+        if connection_params is not None:
+            self._warn_init_param_moved_to_settings("connection_params")
+            if not settings:
+                sample_rate = connection_params.sample_rate
+                encoding = connection_params.encoding
+                default_settings.model = connection_params.speech_model
+                default_settings.formatted_finals = connection_params.formatted_finals
+                default_settings.word_finalization_max_wait_time = (
+                    connection_params.word_finalization_max_wait_time
+                )
+                default_settings.end_of_turn_confidence_threshold = (
+                    connection_params.end_of_turn_confidence_threshold
+                )
+                default_settings.min_turn_silence = connection_params.min_turn_silence
+                default_settings.max_turn_silence = connection_params.max_turn_silence
+                default_settings.keyterms_prompt = connection_params.keyterms_prompt
+                default_settings.prompt = connection_params.prompt
+                default_settings.language_detection = connection_params.language_detection
+                default_settings.format_turns = connection_params.format_turns
+                default_settings.speaker_labels = connection_params.speaker_labels
+                default_settings.vad_threshold = connection_params.vad_threshold
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        # 5. Validate final settings
+        is_u3_pro = default_settings.model == "u3-rt-pro"
+        if not vad_force_turn_endpoint and not is_u3_pro:
+            raise ValueError(
+                f"AssemblyAI turn detection mode (vad_force_turn_endpoint=False) requires "
+                f"u3-rt-pro for SpeechStarted support. Either set "
+                f"vad_force_turn_endpoint=True for {default_settings.model}, "
+                f"or use model='u3-rt-pro'."
+            )
+
+        if default_settings.prompt is not None and default_settings.keyterms_prompt is not None:
+            raise ValueError(
+                "The prompt and keyterms_prompt parameters cannot be used in the same request. "
+                "Please choose either one or the other based on your use case. When you use "
+                "keyterms_prompt, your boosted words are appended to the default prompt automatically. "
+                "Or to boost within prompt: <prompt> + Make sure to boost the words <keyterms> "
+                "in the audio. "
+                "For more info go to: https://www.assemblyai.com/docs/streaming/universal-3-pro"
+            )
+
+        if default_settings.prompt is not None:
+            logger.warning(
+                "Custom prompt detected. Prompting is a beta feature. We recommend testing "
+                "with no prompt first, as this will use our optimized default prompt for "
+                "voice agents. Bad prompts may lead to bad results. If you'd like to create "
+                "your own prompt, check out our prompting guide at: "
+                "https://www.assemblyai.com/docs/streaming/prompting"
+            )
+
+        # 6. Configure pipecat turn mode (mutates default_settings)
+        if vad_force_turn_endpoint:
+            self._configure_pipecat_turn_mode(default_settings, is_u3_pro)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            settings=default_settings,
+            **kwargs,
+        )
 
         self._api_key = api_key
-        self._language = language
         self._api_endpoint_base_url = api_endpoint_base_url
-        self._connection_params = connection_params
         self._vad_force_turn_endpoint = vad_force_turn_endpoint
+        self._should_interrupt = should_interrupt
+        self._speaker_format = speaker_format
+
+        # Init-only audio config (not runtime-updatable)
+        self._encoding = encoding
 
         self._termination_event = asyncio.Event()
         self._received_termination = False
@@ -97,6 +297,54 @@ class AssemblyAISTTService(WebsocketSTTService):
         self._chunk_size_ms = 50
         self._chunk_size_bytes = 0
 
+        self._user_speaking = False
+
+    def _configure_pipecat_turn_mode(self, settings: Settings, is_u3_pro: bool):
+        """Configure settings for Pipecat turn detection mode.
+
+        When vad_force_turn_endpoint is enabled, force AssemblyAI to return
+        finals as fast as possible so Pipecat's smart turn analyzer can decide
+        when the user is done speaking. VAD stop is the absolute ceiling.
+
+        u3-rt-pro:
+        - min_turn_silence defaults to 100ms (user can override)
+        - max_turn_silence is ALWAYS set equal to min_turn_silence
+          to avoid double turn detection (AssemblyAI + Pipecat both analyzing)
+        - If user sets max_turn_silence, it's ignored with a warning
+        - end_of_turn_confidence_threshold: not set (API default)
+
+        universal-streaming-*:
+        - end_of_turn_confidence_threshold=0.0 (disable semantic turn detection)
+        - min_turn_silence=160
+        - max_turn_silence: not set (API default)
+
+        Args:
+            settings: The settings to configure in place.
+            is_u3_pro: Whether using u3-rt-pro model.
+        """
+        if is_u3_pro:
+            # u3-rt-pro: Synchronize max_turn_silence with min_turn_silence
+            min_silence = settings.min_turn_silence
+            if min_silence is None:
+                min_silence = 100
+
+            # Warn if user set max_turn_silence (will be overridden)
+            if settings.max_turn_silence is not None:
+                logger.warning(
+                    f"Your max_turn_silence value ({settings.max_turn_silence}ms) will be "
+                    f"OVERRIDDEN in Pipecat mode (vad_force_turn_endpoint=True). It will be set to "
+                    f"{min_silence}ms (matching min_turn_silence) and SENT to "
+                    f"AssemblyAI to avoid double turn detection. To use your max_turn_silence as-is, "
+                    f"switch to AssemblyAI turn detection mode (vad_force_turn_endpoint=False)."
+                )
+
+            settings.min_turn_silence = min_silence
+            settings.max_turn_silence = min_silence
+        else:
+            # universal-streaming: Different configuration (works differently)
+            settings.end_of_turn_confidence_threshold = 1.0
+            settings.min_turn_silence = 160
+
     def can_generate_metrics(self) -> bool:
         """Check if the service can generate metrics.
 
@@ -105,6 +353,26 @@ class AssemblyAISTTService(WebsocketSTTService):
         """
         return True
 
+    async def _update_settings(self, delta: Settings) -> dict[str, Any]:
+        """Apply a settings delta and reconnect to apply changes.
+
+        Args:
+            delta: A settings delta with updated values.
+
+        Returns:
+            Dict mapping changed field names to their previous values.
+        """
+        changed = await super()._update_settings(delta)
+
+        if not changed:
+            return changed
+
+        # Reconnect to apply updated settings (they become WS query params)
+        await self._disconnect()
+        await self._connect()
+
+        return changed
+
     async def start(self, frame: StartFrame):
         """Start the speech-to-text service.
 
@@ -161,13 +429,14 @@ class AssemblyAISTTService(WebsocketSTTService):
         """
         await super().process_frame(frame, direction)
         if isinstance(frame, VADUserStartedSpeakingFrame):
-            await self.start_ttfb_metrics()
+            pass
         elif isinstance(frame, VADUserStoppedSpeakingFrame):
             if (
                 self._vad_force_turn_endpoint
                 and self._websocket
                 and self._websocket.state is State.OPEN
             ):
+                self.request_finalize()
                 await self._websocket.send(json.dumps({"type": "ForceEndpoint"}))
             await self.start_processing_metrics()
 
@@ -178,16 +447,42 @@ class AssemblyAISTTService(WebsocketSTTService):
 
     def _build_ws_url(self) -> str:
         """Build WebSocket URL with query parameters using urllib.parse.urlencode."""
-        params = {}
-        for k, v in self._connection_params.model_dump().items():
+        s = self._settings
+        params: dict[str, Any] = {}
+
+        # Init-only audio config
+        params["sample_rate"] = self.sample_rate
+        params["encoding"] = self._encoding
+
+        # Map model → speech_model (AssemblyAI API naming)
+        if s.model is not None:
+            params["speech_model"] = s.model
+
+        # Settings fields (skip None values)
+        optional_fields = {
+            "formatted_finals": s.formatted_finals,
+            "word_finalization_max_wait_time": s.word_finalization_max_wait_time,
+            "end_of_turn_confidence_threshold": s.end_of_turn_confidence_threshold,
+            "min_turn_silence": s.min_turn_silence,
+            "max_turn_silence": s.max_turn_silence,
+            "prompt": s.prompt,
+            "language_detection": s.language_detection,
+            "format_turns": s.format_turns,
+            "speaker_labels": s.speaker_labels,
+            "vad_threshold": s.vad_threshold,
+        }
+
+        for k, v in optional_fields.items():
             if v is not None:
-                if k == "keyterms_prompt":
-                    params[k] = json.dumps(v)
-                elif isinstance(v, bool):
+                if isinstance(v, bool):
                     params[k] = str(v).lower()
                 else:
                     params[k] = v
 
+        # Special handling for keyterms_prompt (needs JSON encoding)
+        if s.keyterms_prompt is not None:
+            params["keyterms_prompt"] = json.dumps(s.keyterms_prompt)
+
         if params:
             query_string = urlencode(params)
             return f"{self._api_endpoint_base_url}?{query_string}"
@@ -198,6 +493,8 @@ class AssemblyAISTTService(WebsocketSTTService):
 
         Establishes websocket connection and starts receive task.
         """
+        await super()._connect()
+
         await self._connect_websocket()
 
         if self._websocket and not self._receive_task:
@@ -208,6 +505,8 @@ class AssemblyAISTTService(WebsocketSTTService):
 
         Sends termination message, waits for acknowledgment, and cleans up.
         """
+        await super()._disconnect()
+
         if not self._connected or not self._websocket:
             return
 
@@ -302,6 +601,9 @@ class AssemblyAISTTService(WebsocketSTTService):
         async for message in self._get_websocket():
             try:
                 data = json.loads(message)
+                # Log raw JSON for Turn messages to debug speaker_label
+                if data.get("type") == "Turn":
+                    logger.trace(f"{self} RAW JSON from AssemblyAI: {json.dumps(data, indent=2)}")
                 await self._handle_message(data)
             except json.JSONDecodeError:
                 logger.warning(f"Received non-JSON message: {message}")
@@ -314,6 +616,8 @@ class AssemblyAISTTService(WebsocketSTTService):
             return BeginMessage.model_validate(message)
         elif msg_type == "Turn":
             return TurnMessage.model_validate(message)
+        elif msg_type == "SpeechStarted":
+            return SpeechStartedMessage.model_validate(message)
         elif msg_type == "Termination":
             return TerminationMessage.model_validate(message)
         else:
@@ -330,11 +634,33 @@ class AssemblyAISTTService(WebsocketSTTService):
                 )
             elif isinstance(parsed_message, TurnMessage):
                 await self._handle_transcription(parsed_message)
+            elif isinstance(parsed_message, SpeechStartedMessage):
+                await self._handle_speech_started(parsed_message)
             elif isinstance(parsed_message, TerminationMessage):
                 await self._handle_termination(parsed_message)
         except Exception as e:
             await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
 
+    async def _handle_speech_started(self, message: SpeechStartedMessage):
+        """Handle SpeechStarted event — fast barge-in for AssemblyAI turn detection.
+
+        Broadcasts UserStartedSpeakingFrame to signal the start of user
+        speech, then pushes an interruption to cancel any bot audio.
+        SpeechStarted fires before any transcript arrives, so the turn
+        is cleanly started before any transcription frames are pushed.
+
+        Only applies when using AssemblyAI's built-in turn detection. When using
+        Pipecat turn detection, VAD + smart turn analyzer handle interruptions.
+        """
+        if self._vad_force_turn_endpoint:
+            return  # Pipecat mode: handled by aggregator
+
+        await self.start_processing_metrics()
+        await self.broadcast_frame(UserStartedSpeakingFrame)
+        if self._should_interrupt:
+            await self.broadcast_interruption()
+        self._user_speaking = True
+
     async def _handle_termination(self, message: TerminationMessage):
         """Handle termination message."""
         self._received_termination = True
@@ -347,31 +673,109 @@ class AssemblyAISTTService(WebsocketSTTService):
         await self.push_frame(EndFrame())
 
     async def _handle_transcription(self, message: TurnMessage):
-        """Handle transcription results."""
+        """Handle transcription results with two turn detection modes.
+
+        Pipecat turn detection (vad_force_turn_endpoint=True):
+            - No UserStarted/StoppedSpeakingFrame from STT
+            - end_of_turn → TranscriptionFrame (finalized set by base class
+              if this is a ForceEndpoint response)
+            - else → InterimTranscriptionFrame
+
+        AssemblyAI turn detection (vad_force_turn_endpoint=False):
+            - UserStartedSpeakingFrame on first transcript
+            - end_of_turn → TranscriptionFrame + UserStoppedSpeakingFrame
+            - else → InterimTranscriptionFrame
+        """
         if not message.transcript:
             return
-        await self.stop_ttfb_metrics()
-        if message.end_of_turn and (
-            not self._connection_params.formatted_finals or message.turn_is_formatted
-        ):
-            await self.push_frame(
-                TranscriptionFrame(
-                    message.transcript,
-                    self._user_id,
-                    time_now_iso8601(),
-                    self._language,
-                    message,
+
+        # Use detected language if available with sufficient confidence
+        language = Language.EN
+        if message.language_code and message.language_confidence:
+            if message.language_confidence >= 0.7:
+                language = map_language_from_assemblyai(message.language_code)
+            else:
+                logger.warning(
+                    f"Low language detection confidence ({message.language_confidence:.2f}) "
+                    f"for language '{message.language_code}', falling back to English"
+                )
+
+        # Handle speaker diarization
+        speaker_id = self._user_id
+        transcript_text = message.transcript
+
+        if message.speaker:
+            speaker_id = message.speaker
+            # Format transcript with speaker labels if format string provided
+            if self._speaker_format:
+                transcript_text = self._speaker_format.format(
+                    speaker=message.speaker, text=message.transcript
+                )
+
+        # Determine if this is a final turn from AssemblyAI
+        is_final_turn = message.end_of_turn and (
+            not self._settings.format_turns or message.turn_is_formatted
+        )
+
+        if self._vad_force_turn_endpoint:
+            # --- Pipecat turn detection mode ---
+            # No UserStarted/StoppedSpeakingFrame — VAD + smart turn analyzer handle this
+            if is_final_turn:
+                finalize_confirmed = bool(message.turn_is_formatted)
+                if finalize_confirmed:
+                    self.confirm_finalize()
+                logger.debug(f'{self} Transcript: "{transcript_text}"')
+                await self.push_frame(
+                    TranscriptionFrame(
+                        transcript_text,
+                        speaker_id,
+                        time_now_iso8601(),
+                        language,
+                        message,
+                    )
+                )
+                await self._trace_transcription(transcript_text, True, language)
+                await self.stop_processing_metrics()
+            else:
+                await self.push_frame(
+                    InterimTranscriptionFrame(
+                        transcript_text,
+                        speaker_id,
+                        time_now_iso8601(),
+                        language,
+                        message,
+                    )
                 )
-            )
-            await self._trace_transcription(message.transcript, True, self._language)
-            await self.stop_processing_metrics()
         else:
-            await self.push_frame(
-                InterimTranscriptionFrame(
-                    message.transcript,
-                    self._user_id,
-                    time_now_iso8601(),
-                    self._language,
-                    message,
+            # --- AssemblyAI turn detection mode ---
+            # SpeechStarted always arrives before transcripts with u3-rt-pro,
+            # so UserStartedSpeakingFrame is guaranteed to be broadcast first.
+            if is_final_turn:
+                # AssemblyAI controls finalization, just mark as finalized
+                await self.push_frame(
+                    TranscriptionFrame(
+                        transcript_text,
+                        speaker_id,
+                        time_now_iso8601(),
+                        language,
+                        message,
+                        finalized=True,
+                    )
+                )
+                await self._trace_transcription(transcript_text, True, language)
+                await self.stop_processing_metrics()
+                # AAI is authoritative — emit UserStoppedSpeakingFrame immediately.
+                # broadcast_frame pushes downstream (same queue as TranscriptionFrame
+                # above, so ordering is preserved) and upstream.
+                await self.broadcast_frame(UserStoppedSpeakingFrame)
+                self._user_speaking = False
+            else:
+                await self.push_frame(
+                    InterimTranscriptionFrame(
+                        transcript_text,
+                        speaker_id,
+                        time_now_iso8601(),
+                        language,
+                        message,
+                    )
                 )
-            )

src/pipecat/services/asyncai/tts.py

@@ -9,7 +9,8 @@
 import asyncio
 import base64
 import json
-from typing import AsyncGenerator, Optional
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Optional
 
 import aiohttp
 from loguru import logger
@@ -20,14 +21,13 @@ from pipecat.frames.frames import (
     EndFrame,
     ErrorFrame,
     Frame,
-    InterruptionFrame,
     StartFrame,
     TTSAudioRawFrame,
-    TTSStartedFrame,
     TTSStoppedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.tts_service import InterruptibleTTSService, TTSService
+from pipecat.services.settings import TTSSettings
+from pipecat.services.tts_service import TextAggregationMode, TTSService, WebsocketTTSService
 from pipecat.transcriptions.language import Language, resolve_language
 from pipecat.utils.tracing.service_decorators import traced_tts
 
@@ -72,15 +72,28 @@ def language_to_async_language(language: Language) -> Optional[str]:
     return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
 
 
-class AsyncAITTSService(InterruptibleTTSService):
+@dataclass
+class AsyncAITTSSettings(TTSSettings):
+    """Settings for AsyncAITTSService and AsyncAIHttpTTSService."""
+
+    pass
+
+
+class AsyncAITTSService(WebsocketTTSService):
     """Async TTS service with WebSocket streaming.
 
     Provides text-to-speech using Async's streaming WebSocket API.
     """
 
+    Settings = AsyncAITTSSettings
+    _settings: Settings
+
     class InputParams(BaseModel):
         """Input parameters for Async TTS configuration.
 
+        .. deprecated:: 0.0.105
+            Use ``AsyncAITTSService.Settings`` directly via the ``settings`` parameter instead.
+
         Parameters:
             language: Language to use for synthesis.
         """
@@ -91,15 +104,17 @@ class AsyncAITTSService(InterruptibleTTSService):
         self,
         *,
         api_key: str,
-        voice_id: str,
+        voice_id: Optional[str] = None,
         version: str = "v1",
-        url: str = "wss://api.async.ai/text_to_speech/websocket/ws",
-        model: str = "asyncflow_multilingual_v1.0",
+        url: str = "wss://api.async.com/text_to_speech/websocket/ws",
+        model: Optional[str] = None,
         sample_rate: Optional[int] = None,
         encoding: str = "pcm_s16le",
         container: str = "raw",
         params: Optional[InputParams] = None,
-        aggregate_sentences: Optional[bool] = True,
+        settings: Optional[Settings] = None,
+        aggregate_sentences: Optional[bool] = None,
+        text_aggregation_mode: Optional[TextAggregationMode] = None,
         **kwargs,
     ):
         """Initialize the Async TTS service.
@@ -107,47 +122,97 @@ class AsyncAITTSService(InterruptibleTTSService):
         Args:
             api_key: Async API key.
             voice_id: UUID of the voice to use for synthesis. See docs for a full list:
-                https://docs.async.ai/list-voices-16699698e0
+                https://docs.async.com/list-voices-16699698e0
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AsyncAITTSService.Settings(voice=...)`` instead.
+
             version: Async API version.
             url: WebSocket URL for Async TTS API.
-            model: TTS model to use (e.g., "asyncflow_multilingual_v1.0").
+            model: TTS model to use (e.g., "async_flash_v1.0").
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AsyncAITTSService.Settings(model=...)`` instead.
+
             sample_rate: Audio sample rate.
             encoding: Audio encoding format.
             container: Audio container format.
             params: Additional input parameters for voice customization.
-            aggregate_sentences: Whether to aggregate sentences within the TTSService.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AsyncAITTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
+
+                .. deprecated:: 0.0.104
+                    Use ``text_aggregation_mode`` instead.
+
+            text_aggregation_mode: How to aggregate text before synthesis.
             **kwargs: Additional arguments passed to the parent service.
         """
-        super().__init__(
-            aggregate_sentences=aggregate_sentences,
-            pause_frame_processing=True,
-            push_stop_frames=True,
-            sample_rate=sample_rate,
-            **kwargs,
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="async_flash_v1.0",
+            voice=None,
+            language=None,
         )
 
-        params = params or AsyncAITTSService.InputParams()
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice_id is not None:
+            self._warn_init_param_moved_to_settings("voice_id", "voice")
+            default_settings.voice = voice_id
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.language = params.language
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            aggregate_sentences=aggregate_sentences,
+            text_aggregation_mode=text_aggregation_mode,
+            pause_frame_processing=True,
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            settings=default_settings,
+            **kwargs,
+        )
 
         self._api_key = api_key
         self._api_version = version
         self._url = url
-        self._settings = {
-            "output_format": {
-                "container": container,
-                "encoding": encoding,
-                "sample_rate": 0,
-            },
-            "language": self.language_to_service_language(params.language)
-            if params.language
-            else None,
-        }
 
-        self.set_model_name(model)
-        self.set_voice(voice_id)
+        # Init-only audio format config (not runtime-updatable).
+        self._output_container = container
+        self._output_encoding = encoding
+        self._output_sample_rate = 0  # Set in start()
 
         self._receive_task = None
         self._keepalive_task = None
-        self._started = False
+
+    async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
+        """Apply a settings delta.
+
+        Settings are stored but not applied to the active connection.
+        """
+        changed = await super()._update_settings(delta)
+
+        if not changed:
+            return changed
+
+        self._warn_unhandled_updated_settings(changed)
+
+        return changed
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
@@ -168,8 +233,8 @@ class AsyncAITTSService(InterruptibleTTSService):
         """
         return language_to_async_language(language)
 
-    def _build_msg(self, text: str = "", force: bool = False) -> str:
-        msg = {"transcript": text, "force": force}
+    def _build_msg(self, text: str = "", context_id: str = "", force: bool = False) -> str:
+        msg = {"transcript": text, "context_id": context_id, "force": force}
         return json.dumps(msg)
 
     async def start(self, frame: StartFrame):
@@ -179,7 +244,7 @@ class AsyncAITTSService(InterruptibleTTSService):
             frame: The start frame containing initialization parameters.
         """
         await super().start(frame)
-        self._settings["output_format"]["sample_rate"] = self.sample_rate
+        self._output_sample_rate = self.sample_rate
         await self._connect()
 
     async def stop(self, frame: EndFrame):
@@ -201,6 +266,8 @@ class AsyncAITTSService(InterruptibleTTSService):
         await self._disconnect()
 
     async def _connect(self):
+        await super()._connect()
+
         await self._connect_websocket()
 
         if self._websocket and not self._receive_task:
@@ -210,6 +277,8 @@ class AsyncAITTSService(InterruptibleTTSService):
             self._keepalive_task = self.create_task(self._keepalive_task_handler())
 
     async def _disconnect(self):
+        await super()._disconnect()
+
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
@@ -229,10 +298,14 @@ class AsyncAITTSService(InterruptibleTTSService):
                 f"{self._url}?api_key={self._api_key}&version={self._api_version}"
             )
             init_msg = {
-                "model_id": self._model_name,
-                "voice": {"mode": "id", "id": self._voice_id},
-                "output_format": self._settings["output_format"],
-                "language": self._settings["language"],
+                "model_id": self._settings.model,
+                "voice": {"mode": "id", "id": self._settings.voice},
+                "output_format": {
+                    "container": self._output_container,
+                    "encoding": self._output_encoding,
+                    "sample_rate": self._output_sample_rate,
+                },
+                "language": self._settings.language,
             }
 
             await self._get_websocket().send(json.dumps(init_msg))
@@ -249,12 +322,16 @@ class AsyncAITTSService(InterruptibleTTSService):
 
             if self._websocket:
                 logger.debug("Disconnecting from Async")
+                # Close all contexts and the socket
+                if self.has_active_audio_context():
+                    await self._websocket.send(json.dumps({"terminate": True}))
                 await self._websocket.close()
+                logger.debug("Disconnected from Async")
         except Exception as e:
             await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
         finally:
             self._websocket = None
-            self._started = False
+            await self.remove_active_audio_context()
             await self._call_event_handler("on_disconnected")
 
     def _get_websocket(self):
@@ -262,12 +339,18 @@ class AsyncAITTSService(InterruptibleTTSService):
             return self._websocket
         raise Exception("Websocket not connected")
 
-    async def flush_audio(self):
-        """Flush any pending audio."""
-        if not self._websocket:
+    async def flush_audio(self, context_id: Optional[str] = None):
+        """Flush any pending audio.
+
+        Args:
+            context_id: The specific context to flush. If None, falls back to the
+                currently active context.
+        """
+        flush_id = context_id or self.get_active_audio_context_id()
+        if not flush_id or not self._websocket:
             return
         logger.trace(f"{self}: flushing audio")
-        msg = self._build_msg(text=" ", force=True)
+        msg = self._build_msg(text=" ", context_id=flush_id, force=True)
         await self._websocket.send(msg)
 
     async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
@@ -278,8 +361,6 @@ class AsyncAITTSService(InterruptibleTTSService):
             direction: The direction to push the frame.
         """
         await super().push_frame(frame, direction)
-        if isinstance(frame, (TTSStoppedFrame, InterruptionFrame)):
-            self._started = False
 
     async def _receive_messages(self):
         async for message in self._get_websocket():
@@ -287,41 +368,90 @@ class AsyncAITTSService(InterruptibleTTSService):
             if not msg:
                 continue
 
-            elif msg.get("audio"):
+            received_ctx_id = msg.get("context_id")
+            # Handle final messages first, regardless of context availability
+            # At the moment, this message is received AFTER the close_context message is
+            # sent, so it doesn't serve any functional purpose. For now, we'll just log it.
+            if msg.get("final") is True:
+                logger.trace(f"Received final message for context {received_ctx_id}")
+                continue
+
+            # Check if this message belongs to the current context.
+            if not self.audio_context_available(received_ctx_id):
+                if self.get_active_audio_context_id() == received_ctx_id:
+                    logger.debug(
+                        f"Received a delayed message, recreating the context: {received_ctx_id}"
+                    )
+                    await self.create_audio_context(received_ctx_id)
+                else:
+                    # This can happen if a message is received _after_ we have closed a context
+                    # due to user interruption but _before_ the `isFinal` message for the context
+                    # is received.
+                    logger.debug(f"Ignoring message from unavailable context: {received_ctx_id}")
+                    continue
+
+            if msg.get("audio"):
                 await self.stop_ttfb_metrics()
-                frame = TTSAudioRawFrame(
-                    audio=base64.b64decode(msg["audio"]),
-                    sample_rate=self.sample_rate,
-                    num_channels=1,
-                )
-                await self.push_frame(frame)
-            elif msg.get("error_code"):
-                await self.push_frame(TTSStoppedFrame())
-                await self.stop_all_metrics()
-                await self.push_error(error_msg=f"Error: {msg['message']}")
-            else:
-                await self.push_error(error_msg=f"Unknown message type: {msg}")
+                audio = base64.b64decode(msg["audio"])
+                frame = TTSAudioRawFrame(audio, self.sample_rate, 1, context_id=received_ctx_id)
+                await self.append_to_audio_context(received_ctx_id, frame)
 
     async def _keepalive_task_handler(self):
         """Send periodic keepalive messages to maintain WebSocket connection."""
-        KEEPALIVE_SLEEP = 3
+        KEEPALIVE_SLEEP = 10
         while True:
             await asyncio.sleep(KEEPALIVE_SLEEP)
             try:
                 if self._websocket and self._websocket.state is State.OPEN:
-                    keepalive_message = {"transcript": " "}
-                    logger.trace("Sending keepalive message")
+                    context_id = self.get_active_audio_context_id()
+                    if context_id:
+                        keepalive_message = {
+                            "transcript": " ",
+                            "context_id": context_id,
+                        }
+                        logger.trace("Sending keepalive message")
+                    else:
+                        # It's possible to have a user interruption which clears the context
+                        # without generating a new TTS response. In this case, we'll just send
+                        # an empty message to keep the connection alive.
+                        keepalive_message = {"transcript": " "}
+                        logger.trace("Sending keepalive without context")
                     await self._websocket.send(json.dumps(keepalive_message))
             except websockets.ConnectionClosed as e:
                 logger.warning(f"{self} keepalive error: {e}")
                 break
 
+    async def _close_context(self, context_id: str):
+        # Async AI requires explicit context closure to free server-side resources,
+        # both on interruption and on normal completion.
+        if context_id and self._websocket:
+            try:
+                await self._websocket.send(
+                    json.dumps({"context_id": context_id, "close_context": True, "transcript": ""})
+                )
+            except Exception as e:
+                logger.error(f"{self}: Error closing context {context_id}: {e}")
+
+    async def on_audio_context_interrupted(self, context_id: str):
+        """Close the Async AI context when the bot is interrupted."""
+        await self._close_context(context_id)
+
+    async def on_audio_context_completed(self, context_id: str):
+        """Close the Async AI context after all audio has been played.
+
+        Async AI does not send a server-side signal when a context is
+        exhausted, so Pipecat must explicitly close it with
+        ``close_context: True`` to free server-side resources.
+        """
+        await self._close_context(context_id)
+
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Async API websocket endpoint.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the synthesized speech.
@@ -332,21 +462,14 @@ class AsyncAITTSService(InterruptibleTTSService):
             if not self._websocket or self._websocket.state is State.CLOSED:
                 await self._connect()
 
-            if not self._started:
-                await self.start_ttfb_metrics()
-                yield TTSStartedFrame()
-                self._started = True
-
-            msg = self._build_msg(text=text, force=True)
-
             try:
+                msg = self._build_msg(text=text, force=True, context_id=context_id)
                 await self._get_websocket().send(msg)
                 await self.start_tts_usage_metrics(text)
+
             except Exception as e:
                 yield ErrorFrame(error=f"Unknown error occurred: {e}")
-                yield TTSStoppedFrame()
-                await self._disconnect()
-                await self._connect()
+                yield TTSStoppedFrame(context_id=context_id)
                 return
             yield None
         except Exception as e:
@@ -361,9 +484,15 @@ class AsyncAIHttpTTSService(TTSService):
     connection is not required or desired.
     """
 
+    Settings = AsyncAITTSSettings
+    _settings: Settings
+
     class InputParams(BaseModel):
         """Input parameters for Async API.
 
+        .. deprecated:: 0.0.105
+            Use ``AsyncAIHttpTTSService.Settings`` directly via the ``settings`` parameter instead.
+
         Parameters:
             language: Language to use for synthesis.
         """
@@ -374,15 +503,16 @@ class AsyncAIHttpTTSService(TTSService):
         self,
         *,
         api_key: str,
-        voice_id: str,
+        voice_id: Optional[str] = None,
         aiohttp_session: aiohttp.ClientSession,
-        model: str = "asyncflow_multilingual_v1.0",
-        url: str = "https://api.async.ai",
+        model: Optional[str] = None,
+        url: str = "https://api.async.com",
         version: str = "v1",
         sample_rate: Optional[int] = None,
         encoding: str = "pcm_s16le",
         container: str = "raw",
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Async TTS service.
@@ -390,35 +520,71 @@ class AsyncAIHttpTTSService(TTSService):
         Args:
             api_key: Async API key.
             voice_id: ID of the voice to use for synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AsyncAIHttpTTSService.Settings(voice=...)`` instead.
+
             aiohttp_session: An aiohttp session for making HTTP requests.
-            model: TTS model to use (e.g., "asyncflow_multilingual_v1.0").
+            model: TTS model to use (e.g., "async_flash_v1.0").
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AsyncAIHttpTTSService.Settings(model=...)`` instead.
+
             url: Base URL for Async API.
             version: API version string for Async API.
             sample_rate: Audio sample rate.
             encoding: Audio encoding format.
             container: Audio container format.
             params: Additional input parameters for voice customization.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AsyncAIHttpTTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional arguments passed to the parent TTSService.
         """
-        super().__init__(sample_rate=sample_rate, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="async_flash_v1.0",
+            voice=None,
+            language=None,
+        )
 
-        params = params or AsyncAIHttpTTSService.InputParams()
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice_id is not None:
+            self._warn_init_param_moved_to_settings("voice_id", "voice")
+            default_settings.voice = voice_id
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.language = params.language
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            settings=default_settings,
+            **kwargs,
+        )
 
         self._api_key = api_key
         self._base_url = url
         self._api_version = version
-        self._settings = {
-            "output_format": {
-                "container": container,
-                "encoding": encoding,
-                "sample_rate": 0,
-            },
-            "language": self.language_to_service_language(params.language)
-            if params.language
-            else None,
-        }
-        self.set_voice(voice_id)
-        self.set_model_name(model)
+
+        # Init-only audio format config (not runtime-updatable).
+        self._output_container = container
+        self._output_encoding = encoding
+        self._output_sample_rate = 0  # Set in start()
 
         self._session = aiohttp_session
 
@@ -448,14 +614,15 @@ class AsyncAIHttpTTSService(TTSService):
             frame: The start frame containing initialization parameters.
         """
         await super().start(frame)
-        self._settings["output_format"]["sample_rate"] = self.sample_rate
+        self._output_sample_rate = self.sample_rate
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Async's HTTP streaming API.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the synthesized speech.
@@ -463,16 +630,20 @@ class AsyncAIHttpTTSService(TTSService):
         logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
-            voice_config = {"mode": "id", "id": self._voice_id}
-            await self.start_ttfb_metrics()
+            voice_config = {"mode": "id", "id": self._settings.voice}
+
             payload = {
-                "model_id": self._model_name,
+                "model_id": self._settings.model,
                 "transcript": text,
                 "voice": voice_config,
-                "output_format": self._settings["output_format"],
-                "language": self._settings["language"],
+                "output_format": {
+                    "container": self._output_container,
+                    "encoding": self._output_encoding,
+                    "sample_rate": self._output_sample_rate,
+                },
+                "language": self._settings.language,
             }
-            yield TTSStartedFrame()
+
             headers = {
                 "version": self._api_version,
                 "x-api-key": self._api_key,
@@ -486,7 +657,14 @@ class AsyncAIHttpTTSService(TTSService):
                     await self.push_error(error_msg=f"Async API error: {error_text}")
                     raise Exception(f"Async API returned status {response.status}: {error_text}")
 
-                audio_data = await response.read()
+                # Read streaming bytes; stop TTFB on the *first* received chunk
+                buffer = bytearray()
+                async for chunk in response.content.iter_chunked(64 * 1024):
+                    if not chunk:
+                        continue
+                    await self.stop_ttfb_metrics()
+                    buffer.extend(chunk)
+                audio_data = bytes(buffer)
 
             await self.start_tts_usage_metrics(text)
 
@@ -494,6 +672,7 @@ class AsyncAIHttpTTSService(TTSService):
                 audio=audio_data,
                 sample_rate=self.sample_rate,
                 num_channels=1,
+                context_id=context_id,
             )
 
             yield frame
@@ -502,4 +681,3 @@ class AsyncAIHttpTTSService(TTSService):
             await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
         finally:
             await self.stop_ttfb_metrics()
-            yield TTSStoppedFrame()

src/pipecat/services/aws/llm.py

@@ -18,7 +18,7 @@ import io
 import json
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any, Dict, List, Optional
 
 from loguru import logger
@@ -39,8 +39,6 @@ from pipecat.frames.frames import (
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
     LLMMessagesFrame,
-    LLMTextFrame,
-    LLMUpdateSettingsFrame,
     UserImageRawFrame,
 )
 from pipecat.metrics.metrics import LLMTokenUsage
@@ -57,6 +55,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.llm_service import LLMService
+from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
 from pipecat.utils.tracing.service_decorators import traced_llm
 
 try:
@@ -71,6 +70,23 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+@dataclass
+class AWSBedrockLLMSettings(LLMSettings):
+    """Settings for AWSBedrockLLMService.
+
+    Parameters:
+        stop_sequences: List of strings that stop generation.
+        latency: Performance mode - "standard" or "optimized".
+        additional_model_request_fields: Additional model-specific parameters.
+    """
+
+    stop_sequences: List[str] | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    latency: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    additional_model_request_fields: Dict[str, Any] | _NotGiven = field(
+        default_factory=lambda: NOT_GIVEN
+    )
+
+
 @dataclass
 class AWSBedrockContextAggregatorPair:
     """Container for AWS Bedrock context aggregators.
@@ -355,7 +371,7 @@ class AWSBedrockLLMContext(OpenAILLMContext):
                     tool_result_content = [{"json": content_json}]
                 else:
                     tool_result_content = [{"text": message["content"]}]
-            except:
+            except (json.JSONDecodeError, ValueError, AttributeError):
                 tool_result_content = [{"text": message["content"]}]
 
             return {
@@ -675,7 +691,7 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator):
             frame: The function call result frame to handle.
         """
         if frame.result:
-            result = json.dumps(frame.result)
+            result = json.dumps(frame.result, ensure_ascii=False)
             await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
         else:
             await self._update_function_call_result(
@@ -730,12 +746,19 @@ class AWSBedrockLLMService(LLMService):
     vision capabilities.
     """
 
+    Settings = AWSBedrockLLMSettings
+    _settings: Settings
+
     # Overriding the default adapter to use the Anthropic one.
     adapter_class = AWSBedrockLLMAdapter
 
     class InputParams(BaseModel):
         """Input parameters for AWS Bedrock LLM service.
 
+        .. deprecated:: 0.0.105
+            Use ``AWSBedrockLLMService.Settings`` instead. Pass settings directly via the
+            ``settings`` parameter of :class:`AWSBedrockLLMService`.
+
         Parameters:
             max_tokens: Maximum number of tokens to generate.
             temperature: Sampling temperature between 0.0 and 1.0.
@@ -755,12 +778,14 @@ class AWSBedrockLLMService(LLMService):
     def __init__(
         self,
         *,
-        model: str,
+        model: Optional[str] = None,
         aws_access_key: Optional[str] = None,
         aws_secret_key: Optional[str] = None,
         aws_session_token: Optional[str] = None,
         aws_region: Optional[str] = None,
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
+        stop_sequences: Optional[List[str]] = None,
         client_config: Optional[Config] = None,
         retry_timeout_secs: Optional[float] = 5.0,
         retry_on_timeout: Optional[bool] = False,
@@ -770,19 +795,78 @@ class AWSBedrockLLMService(LLMService):
 
         Args:
             model: The AWS Bedrock model identifier to use.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSBedrockLLMService.Settings(model=...)`` instead.
+
             aws_access_key: AWS access key ID. If None, uses default credentials.
             aws_secret_key: AWS secret access key. If None, uses default credentials.
             aws_session_token: AWS session token for temporary credentials.
             aws_region: AWS region for the Bedrock service.
             params: Model parameters and configuration.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSBedrockLLMService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings for this service.  When both
+                deprecated parameters and *settings* are provided, *settings*
+                values take precedence.
+            stop_sequences: List of strings that stop generation.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSBedrockLLMService.Settings(stop_sequences=...)`` instead.
+
             client_config: Custom boto3 client configuration.
             retry_timeout_secs: Request timeout in seconds for retry logic.
             retry_on_timeout: Whether to retry the request once if it times out.
             **kwargs: Additional arguments passed to parent LLMService.
         """
-        super().__init__(**kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="us.amazon.nova-lite-v1:0",
+            system_instruction=None,
+            max_tokens=None,
+            temperature=None,
+            top_p=None,
+            top_k=None,
+            frequency_penalty=None,
+            presence_penalty=None,
+            seed=None,
+            filter_incomplete_user_turns=False,
+            user_turn_completion_config=None,
+            stop_sequences=None,
+            latency=None,
+            additional_model_request_fields={},
+        )
 
-        params = params or AWSBedrockLLMService.InputParams()
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+        if stop_sequences is not None:
+            self._warn_init_param_moved_to_settings("stop_sequences", "stop_sequences")
+            default_settings.stop_sequences = stop_sequences
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.max_tokens = params.max_tokens
+                default_settings.temperature = params.temperature
+                default_settings.top_p = params.top_p
+                if params.stop_sequences:
+                    default_settings.stop_sequences = params.stop_sequences
+                default_settings.latency = params.latency
+                if isinstance(params.additional_model_request_fields, dict):
+                    default_settings.additional_model_request_fields = (
+                        params.additional_model_request_fields
+                    )
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(settings=default_settings, **kwargs)
 
         # Initialize the AWS Bedrock client
         if not client_config:
@@ -803,20 +887,12 @@ class AWSBedrockLLMService(LLMService):
             "config": client_config,
         }
 
-        self.set_model_name(model)
         self._retry_timeout_secs = retry_timeout_secs
         self._retry_on_timeout = retry_on_timeout
-        self._settings = {
-            "max_tokens": params.max_tokens,
-            "temperature": params.temperature,
-            "top_p": params.top_p,
-            "latency": params.latency,
-            "additional_model_request_fields": params.additional_model_request_fields
-            if isinstance(params.additional_model_request_fields, dict)
-            else {},
-        }
 
-        logger.info(f"Using AWS Bedrock model: {model}")
+        logger.info(f"Using AWS Bedrock model: {self._settings.model}")
+        if self._settings.system_instruction:
+            logger.debug(f"{self}: Using system instruction: {self._settings.system_instruction}")
 
     def can_generate_metrics(self) -> bool:
         """Check if the service can generate usage metrics.
@@ -836,19 +912,30 @@ class AWSBedrockLLMService(LLMService):
             Dictionary containing only the inference parameters that are not None.
         """
         inference_config = {}
-        if self._settings["max_tokens"] is not None:
-            inference_config["maxTokens"] = self._settings["max_tokens"]
-        if self._settings["temperature"] is not None:
-            inference_config["temperature"] = self._settings["temperature"]
-        if self._settings["top_p"] is not None:
-            inference_config["topP"] = self._settings["top_p"]
+        if self._settings.max_tokens is not None:
+            inference_config["maxTokens"] = self._settings.max_tokens
+        if self._settings.temperature is not None:
+            inference_config["temperature"] = self._settings.temperature
+        if self._settings.top_p is not None:
+            inference_config["topP"] = self._settings.top_p
+        if self._settings.stop_sequences:
+            inference_config["stopSequences"] = self._settings.stop_sequences
         return inference_config
 
-    async def run_inference(self, context: LLMContext | OpenAILLMContext) -> Optional[str]:
+    async def run_inference(
+        self,
+        context: LLMContext | OpenAILLMContext,
+        max_tokens: Optional[int] = None,
+        system_instruction: Optional[str] = None,
+    ) -> Optional[str]:
         """Run a one-shot, out-of-band (i.e. out-of-pipeline) inference with the given LLM context.
 
         Args:
             context: The LLM context containing conversation history.
+            max_tokens: Optional maximum number of tokens to generate. If provided,
+                overrides the service's default max_tokens setting.
+            system_instruction: Optional system instruction to use for this inference.
+                If provided, overrides any system instruction in the context.
 
         Returns:
             The LLM's response as a string, or None if no response is generated.
@@ -865,13 +952,26 @@ class AWSBedrockLLMService(LLMService):
             messages = context.messages
             system = getattr(context, "system", None)  # [{"text": "system message"}]
 
+        # Override system instruction if provided
+        if system_instruction is not None:
+            if system:
+                logger.warning(
+                    f"{self}: Both system_instruction and a system message in context are set."
+                    " Using system_instruction."
+                )
+            system = [{"text": system_instruction}]
+
         # Prepare request parameters using the same method as streaming
         inference_config = self._build_inference_config()
 
+        # Override maxTokens if provided
+        if max_tokens is not None:
+            inference_config["maxTokens"] = max_tokens
+
         request_params = {
-            "modelId": self.model_name,
+            "modelId": self._settings.model,
             "messages": messages,
-            "additionalModelRequestFields": self._settings["additional_model_request_fields"],
+            "additionalModelRequestFields": self._settings.additional_model_request_fields,
         }
 
         if inference_config:
@@ -986,7 +1086,14 @@ class AWSBedrockLLMService(LLMService):
         # Universal LLMContext
         if isinstance(context, LLMContext):
             adapter: AWSBedrockLLMAdapter = self.get_llm_adapter()
-            params = adapter.get_llm_invocation_params(context)
+            params: AWSBedrockLLMInvocationParams = adapter.get_llm_invocation_params(context)
+            if self._settings.system_instruction:
+                if params["system"]:
+                    logger.warning(
+                        f"{self}: Both system_instruction and a system message in context are"
+                        " set. Using system_instruction."
+                    )
+                params["system"] = [{"text": self._settings.system_instruction}]
             return params
 
         # AWS Bedrock-specific context
@@ -1026,9 +1133,9 @@ class AWSBedrockLLMService(LLMService):
 
             # Prepare request parameters
             request_params = {
-                "modelId": self.model_name,
+                "modelId": self._settings.model,
                 "messages": messages,
-                "additionalModelRequestFields": self._settings["additional_model_request_fields"],
+                "additionalModelRequestFields": self._settings.additional_model_request_fields,
             }
 
             # Only add inference config if it has parameters
@@ -1073,8 +1180,8 @@ class AWSBedrockLLMService(LLMService):
                 request_params["toolConfig"] = tool_config
 
             # Add performance config if latency is specified
-            if self._settings["latency"] in ["standard", "optimized"]:
-                request_params["performanceConfig"] = {"latency": self._settings["latency"]}
+            if self._settings.latency in ["standard", "optimized"]:
+                request_params["performanceConfig"] = {"latency": self._settings.latency}
 
             # Log request params with messages redacted for logging
             if isinstance(context, LLMContext):
@@ -1107,7 +1214,7 @@ class AWSBedrockLLMService(LLMService):
                     if "contentBlockDelta" in event:
                         delta = event["contentBlockDelta"]["delta"]
                         if "text" in delta:
-                            await self.push_frame(LLMTextFrame(delta["text"]))
+                            await self._push_llm_text(delta["text"])
                             completion_tokens_estimate += self._estimate_tokens(delta["text"])
                         elif "toolUse" in delta and "input" in delta["toolUse"]:
                             # Handle partial JSON for tool use
@@ -1199,8 +1306,6 @@ class AWSBedrockLLMService(LLMService):
             # NOTE: LLMMessagesFrame is deprecated, so we don't support the newer universal
             # LLMContext with it
             context = AWSBedrockLLMContext.from_messages(frame.messages)
-        elif isinstance(frame, LLMUpdateSettingsFrame):
-            await self._update_settings(frame.settings)
         else:
             await self.push_frame(frame, direction)
 

src/pipecat/services/aws/nova_sonic/llm.py

@@ -16,7 +16,7 @@ import json
 import time
 import uuid
 import wave
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from enum import Enum
 from importlib.resources import files
 from typing import Any, List, Optional
@@ -27,8 +27,8 @@ from pydantic import BaseModel, Field
 from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.adapters.services.aws_nova_sonic_adapter import AWSNovaSonicLLMAdapter, Role
 from pipecat.frames.frames import (
+    AggregatedTextFrame,
     AggregationType,
-    BotStoppedSpeakingFrame,
     CancelFrame,
     EndFrame,
     Frame,
@@ -38,6 +38,7 @@ from pipecat.frames.frames import (
     LLMContextFrame,
     LLMFullResponseEndFrame,
     LLMFullResponseStartFrame,
+    LLMTextFrame,
     StartFrame,
     TranscriptionFrame,
     TTSAudioRawFrame,
@@ -59,6 +60,7 @@ from pipecat.processors.aggregators.openai_llm_context import (
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.llm_service import LLMService
+from pipecat.services.settings import NOT_GIVEN, LLMSettings, _NotGiven
 from pipecat.utils.time import time_now_iso8601
 
 try:
@@ -147,6 +149,10 @@ class CurrentContent:
 class Params(BaseModel):
     """Configuration parameters for AWS Nova Sonic.
 
+    .. deprecated:: 0.0.105
+        Use ``settings=AWSNovaSonicLLMService.Settings(...)`` for inference settings
+        and ``audio_config=AudioConfig(...)`` for audio configuration.
+
     Parameters:
         input_sample_rate: Audio input sample rate in Hz.
         input_sample_size: Audio input sample size in bits.
@@ -183,6 +189,55 @@ class Params(BaseModel):
     # Turn-taking
     endpointing_sensitivity: Optional[str] = Field(default=None)
 
+    @property
+    def audio_config(self) -> "AudioConfig":
+        """Return an ``AudioConfig`` populated from this instance's audio fields."""
+        return AudioConfig(
+            input_sample_rate=self.input_sample_rate,
+            input_sample_size=self.input_sample_size,
+            input_channel_count=self.input_channel_count,
+            output_sample_rate=self.output_sample_rate,
+            output_sample_size=self.output_sample_size,
+            output_channel_count=self.output_channel_count,
+        )
+
+
+class AudioConfig(BaseModel):
+    """Audio configuration for AWS Nova Sonic.
+
+    Parameters:
+        input_sample_rate: Audio input sample rate in Hz.
+        input_sample_size: Audio input sample size in bits.
+        input_channel_count: Number of input audio channels.
+        output_sample_rate: Audio output sample rate in Hz.
+        output_sample_size: Audio output sample size in bits.
+        output_channel_count: Number of output audio channels.
+    """
+
+    # Input
+    input_sample_rate: Optional[int] = Field(default=16000)
+    input_sample_size: Optional[int] = Field(default=16)
+    input_channel_count: Optional[int] = Field(default=1)
+
+    # Output
+    output_sample_rate: Optional[int] = Field(default=24000)
+    output_sample_size: Optional[int] = Field(default=16)
+    output_channel_count: Optional[int] = Field(default=1)
+
+
+@dataclass
+class AWSNovaSonicLLMSettings(LLMSettings):
+    """Settings for AWSNovaSonicLLMService.
+
+    Parameters:
+        voice: Voice identifier for speech synthesis.
+        endpointing_sensitivity: Controls how quickly Nova Sonic decides the
+            user has stopped speaking. Can be "LOW", "MEDIUM", or "HIGH".
+    """
+
+    voice: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    endpointing_sensitivity: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
 
 class AWSNovaSonicLLMService(LLMService):
     """AWS Nova Sonic speech-to-speech LLM service.
@@ -191,6 +246,9 @@ class AWSNovaSonicLLMService(LLMService):
     and function calling capabilities using AWS Nova Sonic model.
     """
 
+    Settings = AWSNovaSonicLLMSettings
+    _settings: Settings
+
     # Override the default adapter to use the AWSNovaSonicLLMAdapter one
     adapter_class = AWSNovaSonicLLMAdapter
 
@@ -204,6 +262,8 @@ class AWSNovaSonicLLMService(LLMService):
         model: str = "amazon.nova-2-sonic-v1:0",
         voice_id: str = "matthew",
         params: Optional[Params] = None,
+        audio_config: Optional[AudioConfig] = None,
+        settings: Optional[Settings] = None,
         system_instruction: Optional[str] = None,
         tools: Optional[ToolsSchema] = None,
         send_transcription_frames: bool = True,
@@ -220,13 +280,35 @@ class AWSNovaSonicLLMService(LLMService):
                 - Nova 2 Sonic (the default model): "us-east-1", "us-west-2", "ap-northeast-1"
                 - Nova Sonic (the older model): "us-east-1", "ap-northeast-1"
             model: Model identifier. Defaults to "amazon.nova-2-sonic-v1:0".
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSNovaSonicLLMService.Settings(model=...)`` instead.
+
             voice_id: Voice ID for speech synthesis.
                 Note that some voices are designed for use with a specific language.
                 Options:
                 - Nova 2 Sonic (the default model): see https://docs.aws.amazon.com/nova/latest/nova2-userguide/sonic-language-support.html
                 - Nova Sonic (the older model): see https://docs.aws.amazon.com/nova/latest/userguide/available-voices.html.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSNovaSonicLLMService.Settings(voice=...)`` instead.
+
             params: Model parameters for audio configuration and inference.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSNovaSonicLLMService.Settings(...)`` for inference
+                    settings and ``audio_config=AudioConfig(...)`` for audio
+                    configuration.
+
+            audio_config: Audio configuration (sample rates, sample sizes,
+                channel counts). If not provided, defaults are used.
+            settings: AWS Nova Sonic LLM settings. If provided together with
+                deprecated top-level parameters, the ``settings`` values take
+                precedence.
             system_instruction: System-level instruction for the model.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSNovaSonicLLMService.Settings(system_instruction=...)`` instead.
             tools: Available tools/functions for the model to use.
             send_transcription_frames: Whether to emit transcription frames.
 
@@ -236,28 +318,86 @@ class AWSNovaSonicLLMService(LLMService):
 
             **kwargs: Additional arguments passed to the parent LLMService.
         """
-        super().__init__(**kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="amazon.nova-2-sonic-v1:0",
+            system_instruction=None,
+            voice="matthew",
+            temperature=0.7,
+            max_tokens=1024,
+            top_p=0.9,
+            top_k=None,
+            frequency_penalty=None,
+            presence_penalty=None,
+            seed=None,
+            filter_incomplete_user_turns=False,
+            user_turn_completion_config=None,
+            endpointing_sensitivity=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model != "amazon.nova-2-sonic-v1:0":
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+        if voice_id != "matthew":
+            self._warn_init_param_moved_to_settings("voice_id", "voice")
+            default_settings.voice = voice_id
+        if system_instruction is not None:
+            self._warn_init_param_moved_to_settings("system_instruction", "system_instruction")
+            default_settings.system_instruction = system_instruction
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            import warnings
+
+            with warnings.catch_warnings():
+                warnings.simplefilter("always")
+                warnings.warn(
+                    "The `params` parameter is deprecated. "
+                    "Use `settings=self.Settings(...)` for inference settings "
+                    "(temperature, max_tokens, top_p, endpointing_sensitivity) "
+                    "and `audio_config=AudioConfig(...)` for audio configuration "
+                    "(sample rates, sample sizes, channel counts).",
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
+            if not settings:
+                default_settings.temperature = params.temperature
+                default_settings.max_tokens = params.max_tokens
+                default_settings.top_p = params.top_p
+                default_settings.endpointing_sensitivity = params.endpointing_sensitivity
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            settings=default_settings,
+            **kwargs,
+        )
         self._secret_access_key = secret_access_key
         self._access_key_id = access_key_id
         self._session_token = session_token
         self._region = region
-        self._model = model
         self._client: Optional[BedrockRuntimeClient] = None
-        self._voice_id = voice_id
-        self._params = params or Params()
-        self._system_instruction = system_instruction
+
+        # Audio I/O config (hardware settings, not runtime-tunable)
+        # Priority: audio_config > params (deprecated) > defaults
+        self._audio_config = audio_config or (
+            params.audio_config if params is not None else AudioConfig()
+        )
         self._tools = tools
 
         # Validate endpointing_sensitivity parameter
         if (
-            self._params.endpointing_sensitivity
+            self._settings.endpointing_sensitivity
             and not self._is_endpointing_sensitivity_supported()
         ):
             logger.warning(
-                f"endpointing_sensitivity is not supported for model '{model}' and will be ignored. "
+                f"endpointing_sensitivity is not supported for model '{self._settings.model}' and will be ignored. "
                 "This parameter is only supported starting with Nova 2 Sonic (amazon.nova-2-sonic-v1:0)."
             )
-            self._params.endpointing_sensitivity = None
+            self._settings.endpointing_sensitivity = None
 
         if not send_transcription_frames:
             import warnings
@@ -284,22 +424,44 @@ class AWSNovaSonicLLMService(LLMService):
         self._input_audio_content_name: Optional[str] = None
         self._content_being_received: Optional[CurrentContent] = None
         self._assistant_is_responding = False
-        self._may_need_repush_assistant_text = False
         self._ready_to_send_context = False
-        self._handling_bot_stopped_speaking = False
         self._triggering_assistant_response = False
         self._waiting_for_trigger_transcription = False
         self._disconnecting = False
         self._connected_time: Optional[float] = None
         self._wants_connection = False
         self._user_text_buffer = ""
-        self._assistant_text_buffer = ""
         self._completed_tool_calls = set()
+        self._audio_input_started = False
+        self._pending_speculative_text: Optional[str] = None
 
         file_path = files("pipecat.services.aws.nova_sonic").joinpath("ready.wav")
         with wave.open(file_path.open("rb"), "rb") as wav_file:
             self._assistant_response_trigger_audio = wav_file.readframes(wav_file.getnframes())
 
+    #
+    # settings
+    #
+
+    async def _update_settings(self, delta: Settings) -> dict[str, Any]:
+        """Apply a settings delta.
+
+        Settings are stored but not applied to the active connection.
+        """
+        changed = await super()._update_settings(delta)
+
+        if not changed:
+            return changed
+
+        # TODO: someday we could reconnect here to apply updated settings.
+        # Code might look something like the below:
+        # await self._disconnect()
+        # await self._start_connecting()
+
+        self._warn_unhandled_updated_settings(changed)
+
+        return changed
+
     #
     # standard AIService frame handling
     #
@@ -341,11 +503,13 @@ class AWSNovaSonicLLMService(LLMService):
     async def reset_conversation(self):
         """Reset the conversation state while preserving context.
 
-        Handles bot stopped speaking event, disconnects from the service,
-        and reconnects with the preserved context.
+        Cleans up any in-progress assistant response, disconnects from the
+        service, and reconnects with the preserved context.
         """
         logger.debug("Resetting conversation")
-        await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False)
+        if self._assistant_is_responding:
+            self._assistant_is_responding = False
+            await self._report_assistant_response_ended()
 
         # Grab context to carry through disconnect/reconnect
         context = self._context
@@ -376,8 +540,6 @@ class AWSNovaSonicLLMService(LLMService):
             await self._handle_context(context)
         elif isinstance(frame, InputAudioRawFrame):
             await self._handle_input_audio_frame(frame)
-        elif isinstance(frame, BotStoppedSpeakingFrame):
-            await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=True)
         elif isinstance(frame, InterruptionFrame):
             await self._handle_interruption_frame()
 
@@ -405,49 +567,8 @@ class AWSNovaSonicLLMService(LLMService):
 
         await self._send_user_audio_event(frame.audio)
 
-    async def _handle_bot_stopped_speaking(self, delay_to_catch_trailing_assistant_text: bool):
-        # Protect against back-to-back BotStoppedSpeaking calls, which I've observed
-        if self._handling_bot_stopped_speaking:
-            return
-        self._handling_bot_stopped_speaking = True
-
-        async def finalize_assistant_response():
-            if self._assistant_is_responding:
-                # Consider the assistant finished with their response (possibly after a short delay,
-                # to allow for any trailing FINAL assistant text block to come in that need to make
-                # it into context).
-                #
-                # TODO: ideally we could base this solely on the LLM output events, but I couldn't
-                # figure out a reliable way to determine when we've gotten our last FINAL text block
-                # after the LLM is done talking.
-                #
-                # First I looked at stopReason, but it doesn't seem like the last FINAL text block
-                # is reliably marked END_TURN (sometimes the *first* one is, but not the last...
-                # bug?)
-                #
-                # Then I considered schemes where we tally or match up SPECULATIVE text blocks with
-                # FINAL text blocks to know how many or which FINAL blocks to expect, but user
-                # interruptions throw a wrench in these schemes: depending on the exact timing of
-                # the interruption, we should or shouldn't expect some FINAL blocks.
-                if delay_to_catch_trailing_assistant_text:
-                    # This delay length is a balancing act between "catching" trailing assistant
-                    # text that is quite delayed but not waiting so long that user text comes in
-                    # first and results in a bit of context message order scrambling.
-                    await asyncio.sleep(1.25)
-                self._assistant_is_responding = False
-                await self._report_assistant_response_ended()
-
-            self._handling_bot_stopped_speaking = False
-
-        # Finalize the assistant response, either now or after a delay
-        if delay_to_catch_trailing_assistant_text:
-            self.create_task(finalize_assistant_response())
-        else:
-            await finalize_assistant_response()
-
     async def _handle_interruption_frame(self):
-        if self._assistant_is_responding:
-            self._may_need_repush_assistant_text = True
+        pass
 
     #
     # LLM communication: lifecycle
@@ -470,7 +591,7 @@ class AWSNovaSonicLLMService(LLMService):
 
             # Start the bidirectional stream
             self._stream = await self._client.invoke_model_with_bidirectional_stream(
-                InvokeModelWithBidirectionalStreamOperationInput(model_id=self._model)
+                InvokeModelWithBidirectionalStreamOperationInput(model_id=self._settings.model)
             )
 
             # Send session start event
@@ -521,24 +642,40 @@ class AWSNovaSonicLLMService(LLMService):
         await self._send_prompt_start_event(tools)
 
         # Send system instruction.
-        # Instruction from context takes priority over self._system_instruction.
+        # Instruction from context takes priority over self._settings.system_instruction.
         system_instruction = (
             llm_connection_params["system_instruction"]
             if llm_connection_params["system_instruction"]
-            else self._system_instruction
+            else self._settings.system_instruction
         )
         logger.debug(f"Using system instruction: {system_instruction}")
         if system_instruction:
             await self._send_text_event(text=system_instruction, role=Role.SYSTEM)
 
-        # Send conversation history
-        for message in llm_connection_params["messages"]:
+        # Send conversation history (except for the last message if it's from the
+        # user, which we'll send as interactive after starting audio input)
+        messages = llm_connection_params["messages"]
+        last_user_message = None
+        for i, message in enumerate(messages):
             # logger.debug(f"Seeding conversation history with message: {message}")
-            await self._send_text_event(text=message.text, role=message.role)
+            is_last_message = i == len(messages) - 1
+            if is_last_message and message.role == Role.USER:
+                # Save for sending after audio input starts
+                last_user_message = message
+            else:
+                await self._send_text_event(text=message.text, role=message.role)
 
         # Start audio input
         await self._send_audio_input_start_event()
 
+        # Now send the last user message as interactive to trigger bot response
+        if last_user_message:
+            # logger.debug(
+            #     f"Sending last user message as interactive to trigger bot response: {last_user_message}")
+            await self._send_text_event(
+                text=last_user_message.text, role=last_user_message.role, interactive=True
+            )
+
         # Start receiving events
         self._receive_task = self.create_task(self._receive_task_handler())
 
@@ -591,16 +728,15 @@ class AWSNovaSonicLLMService(LLMService):
             self._input_audio_content_name = None
             self._content_being_received = None
             self._assistant_is_responding = False
-            self._may_need_repush_assistant_text = False
             self._ready_to_send_context = False
-            self._handling_bot_stopped_speaking = False
             self._triggering_assistant_response = False
             self._waiting_for_trigger_transcription = False
             self._disconnecting = False
             self._connected_time = None
             self._user_text_buffer = ""
-            self._assistant_text_buffer = ""
             self._completed_tool_calls = set()
+            self._audio_input_started = False
+            self._pending_speculative_text = None
 
             logger.info("Finished disconnecting")
         except Exception as e:
@@ -620,7 +756,7 @@ class AWSNovaSonicLLMService(LLMService):
 
     def _is_first_generation_sonic_model(self) -> bool:
         # Nova Sonic (the older model) is identified by "amazon.nova-sonic-v1:0"
-        return self._model == "amazon.nova-sonic-v1:0"
+        return self._settings.model == "amazon.nova-sonic-v1:0"
 
     def _is_endpointing_sensitivity_supported(self) -> bool:
         # endpointing_sensitivity is only supported with Nova 2 Sonic (and,
@@ -639,9 +775,9 @@ class AWSNovaSonicLLMService(LLMService):
         turn_detection_config = (
             f""",
               "turnDetectionConfiguration": {{
-                "endpointingSensitivity": "{self._params.endpointing_sensitivity}"
+                "endpointingSensitivity": "{self._settings.endpointing_sensitivity}"
               }}"""
-            if self._params.endpointing_sensitivity
+            if self._settings.endpointing_sensitivity
             else ""
         )
 
@@ -650,9 +786,9 @@ class AWSNovaSonicLLMService(LLMService):
           "event": {{
             "sessionStart": {{
               "inferenceConfiguration": {{
-                "maxTokens": {self._params.max_tokens},
-                "topP": {self._params.top_p},
-                "temperature": {self._params.temperature}
+                "maxTokens": {self._settings.max_tokens},
+                "topP": {self._settings.top_p},
+                "temperature": {self._settings.temperature}
               }}{turn_detection_config}
             }}
           }}
@@ -687,10 +823,10 @@ class AWSNovaSonicLLMService(LLMService):
               }},
               "audioOutputConfiguration": {{
                 "mediaType": "audio/lpcm",
-                "sampleRateHertz": {self._params.output_sample_rate},
-                "sampleSizeBits": {self._params.output_sample_size},
-                "channelCount": {self._params.output_channel_count},
-                "voiceId": "{self._voice_id}",
+                "sampleRateHertz": {self._audio_config.output_sample_rate},
+                "sampleSizeBits": {self._audio_config.output_sample_size},
+                "channelCount": {self._audio_config.output_channel_count},
+                "voiceId": "{self._settings.voice}",
                 "encoding": "base64",
                 "audioType": "SPEECH"
               }}{tools_config}
@@ -715,9 +851,9 @@ class AWSNovaSonicLLMService(LLMService):
                     "role": "USER",
                     "audioInputConfiguration": {{
                         "mediaType": "audio/lpcm",
-                        "sampleRateHertz": {self._params.input_sample_rate},
-                        "sampleSizeBits": {self._params.input_sample_size},
-                        "channelCount": {self._params.input_channel_count},
+                        "sampleRateHertz": {self._audio_config.input_sample_rate},
+                        "sampleSizeBits": {self._audio_config.input_sample_size},
+                        "channelCount": {self._audio_config.input_channel_count},
                         "audioType": "SPEECH",
                         "encoding": "base64"
                     }}
@@ -726,8 +862,18 @@ class AWSNovaSonicLLMService(LLMService):
         }}
         '''
         await self._send_client_event(audio_content_start)
+        self._audio_input_started = True
 
-    async def _send_text_event(self, text: str, role: Role):
+    async def _send_text_event(self, text: str, role: Role, interactive: bool = False):
+        """Send a text event to the LLM.
+
+        Args:
+            text: The text content to send.
+            role: The role associated with the text (e.g., USER, ASSISTANT, SYSTEM).
+            interactive: Whether the content is interactive. Defaults to False.
+                False: conversation history or system instruction, sent prior to interactive audio
+                True: text input sent during (or at the start of) interactive audio
+        """
         if not self._stream or not self._prompt_name or not text:
             return
 
@@ -740,7 +886,7 @@ class AWSNovaSonicLLMService(LLMService):
                     "promptName": "{self._prompt_name}",
                     "contentName": "{content_name}",
                     "type": "TEXT",
-                    "interactive": true,
+                    "interactive": {json.dumps(interactive)},
                     "role": "{role.value}",
                     "textInputConfiguration": {{
                         "mediaType": "text/plain"
@@ -778,7 +924,7 @@ class AWSNovaSonicLLMService(LLMService):
         await self._send_client_event(text_content_end)
 
     async def _send_user_audio_event(self, audio: bytes):
-        if not self._stream:
+        if not self._stream or not self._audio_input_started:
             return
 
         blob = base64.b64encode(audio)
@@ -853,7 +999,9 @@ class AWSNovaSonicLLMService(LLMService):
                     "toolResult": {
                         "promptName": self._prompt_name,
                         "contentName": content_name,
-                        "content": json.dumps(result) if isinstance(result, dict) else result,
+                        "content": json.dumps(result, ensure_ascii=False)
+                        if isinstance(result, dict)
+                        else result,
                     }
                 }
             }
@@ -960,10 +1108,11 @@ class AWSNovaSonicLLMService(LLMService):
         self._content_being_received = content
 
         if content.role == Role.ASSISTANT:
-            if content.type == ContentType.AUDIO:
-                # Note that an assistant response can comprise of multiple audio blocks
-                if not self._assistant_is_responding:
-                    # The assistant has started responding.
+            if content.type == ContentType.TEXT:
+                if (
+                    content.text_stage == TextStage.SPECULATIVE
+                    and not self._assistant_is_responding
+                ):
                     self._assistant_is_responding = True
                     await self._report_user_transcription_ended()  # Consider user turn over
                     await self._report_assistant_response_started()
@@ -990,8 +1139,8 @@ class AWSNovaSonicLLMService(LLMService):
         audio = base64.b64decode(audio_content)
         frame = TTSAudioRawFrame(
             audio=audio,
-            sample_rate=self._params.output_sample_rate,
-            num_channels=self._params.output_channel_count,
+            sample_rate=self._audio_config.output_sample_rate,
+            num_channels=self._audio_config.output_channel_count,
         )
         await self.push_frame(frame)
 
@@ -1039,18 +1188,30 @@ class AWSNovaSonicLLMService(LLMService):
 
         if content.role == Role.ASSISTANT:
             if content.type == ContentType.TEXT:
-                # Ignore non-final text, and the "interrupted" message (which isn't meaningful text)
-                if content.text_stage == TextStage.FINAL and stop_reason != "INTERRUPTED":
-                    if self._assistant_is_responding:
-                        # Text added to the ongoing assistant response
-                        await self._report_assistant_response_text_added(content.text_content)
+                if stop_reason != "INTERRUPTED":
+                    if content.text_stage == TextStage.SPECULATIVE:
+                        await self._report_llm_text(content.text_content)
+                elif self._assistant_is_responding:
+                    # TEXT INTERRUPTED with no audio means the user interrupted
+                    # before audio started. End the response here since no AUDIO
+                    # contentEnd will arrive.
+                    self._assistant_is_responding = False
+                    await self._report_assistant_response_ended()
+            elif content.type == ContentType.AUDIO:
+                # Emit deferred TTSTextFrame after all audio chunks have been sent
+                await self._report_tts_text()
+                if stop_reason in ("END_TURN", "INTERRUPTED"):
+                    # END_TURN: normal completion. INTERRUPTED: user interrupted
+                    # mid-audio. Both mean no more audio for this turn.
+                    self._assistant_is_responding = False
+                    await self._report_assistant_response_ended()
         elif content.role == Role.USER:
             if content.type == ContentType.TEXT:
                 if content.text_stage == TextStage.FINAL:
                     # User transcription text added
                     await self._report_user_transcription_text_added(content.text_content)
 
-    async def _handle_completion_end_event(self, event_json):
+    async def _handle_completion_end_event(self, _):
         pass
 
     #
@@ -1063,31 +1224,40 @@ class AWSNovaSonicLLMService(LLMService):
 
     async def _report_assistant_response_started(self):
         logger.debug("Assistant response started")
-
-        # Report the start of the assistant response.
         await self.push_frame(LLMFullResponseStartFrame())
 
         # Report that equivalent of TTS (this is a speech-to-speech model) started
         await self.push_frame(TTSStartedFrame())
 
-    async def _report_assistant_response_text_added(self, text):
-        if not self._context:  # should never happen
-            return
+    async def _report_llm_text(self, text):
+        """Push speculative assistant text and defer TTSTextFrame.
 
-        logger.debug(f"Assistant response text added: {text}")
+        Speculative text arrives before each audio chunk, providing real-time
+        text that is synchronized with what the bot is saying. LLMTextFrame and
+        AggregatedTextFrame are pushed immediately for real-time text display.
+        TTSTextFrame emission is deferred to audio contentEnd so it aligns with
+        audio playout timing.
+        """
+        logger.debug(f"Assistant speculative text: {text}")
 
-        # Report the text of the assistant response.
-        frame = TTSTextFrame(text, aggregated_by=AggregationType.SENTENCE)
-        frame.includes_inter_frame_spaces = True
-        await self.push_frame(frame)
+        llm_text_frame = LLMTextFrame(text)
+        llm_text_frame.append_to_context = False
+        await self.push_frame(llm_text_frame)
 
-        # HACK: here we're also buffering the assistant text ourselves as a
-        # backup rather than relying solely on the assistant context aggregator
-        # to do it, because the text arrives from Nova Sonic only after all the
-        # assistant audio frames have been pushed, meaning that if an
-        # interruption frame were to arrive we would lose all of it (the text
-        # frames sitting in the queue would be wiped).
-        self._assistant_text_buffer += text
+        aggregated_text_frame = AggregatedTextFrame(text, aggregated_by=AggregationType.SENTENCE)
+        aggregated_text_frame.append_to_context = False
+        await self.push_frame(aggregated_text_frame)
+
+        self._pending_speculative_text = text
+
+    async def _report_tts_text(self):
+        if self._pending_speculative_text:
+            tts_text_frame = TTSTextFrame(
+                self._pending_speculative_text, aggregated_by=AggregationType.SENTENCE
+            )
+            tts_text_frame.includes_inter_frame_spaces = True
+            await self.push_frame(tts_text_frame)
+            self._pending_speculative_text = None
 
     async def _report_assistant_response_ended(self):
         if not self._context:  # should never happen
@@ -1095,39 +1265,12 @@ class AWSNovaSonicLLMService(LLMService):
 
         logger.debug("Assistant response ended")
 
-        # If an interruption frame arrived while the assistant was responding
-        # we may have lost all of the assistant text (see HACK, above), so
-        # re-push it downstream to the aggregator now.
-        if self._may_need_repush_assistant_text:
-            # Just in case, check that assistant text hasn't already made it
-            # into the context (sometimes it does, despite the interruption).
-            messages = self._context.get_messages()
-            last_message = messages[-1] if messages else None
-            if (
-                not last_message
-                or last_message.get("role") != "assistant"
-                or last_message.get("content") != self._assistant_text_buffer
-            ):
-                # We also need to re-push the LLMFullResponseStartFrame since the
-                # TTSTextFrame would be ignored otherwise (the interruption frame
-                # would have cleared the assistant aggregator state).
-                await self.push_frame(LLMFullResponseStartFrame())
-                frame = TTSTextFrame(
-                    self._assistant_text_buffer, aggregated_by=AggregationType.SENTENCE
-                )
-                frame.includes_inter_frame_spaces = True
-                await self.push_frame(frame)
-            self._may_need_repush_assistant_text = False
-
         # Report the end of the assistant response.
         await self.push_frame(LLMFullResponseEndFrame())
 
         # Report that equivalent of TTS (this is a speech-to-speech model) stopped.
         await self.push_frame(TTSStoppedFrame())
 
-        # Clear out the buffered assistant text
-        self._assistant_text_buffer = ""
-
     #
     # user transcription reporting
     #
@@ -1157,6 +1300,12 @@ class AWSNovaSonicLLMService(LLMService):
         if not self._context:  # should never happen
             return
 
+        # Nothing to report if no user speech was transcribed (e.g. the prompt
+        # was text-only, which is the case on the first user turn when the bot
+        # starts the conversation).
+        if not self._user_text_buffer:
+            return
+
         logger.debug(f"User transcription ended")
 
         # Report to the upstream user context aggregator that some new user
@@ -1187,7 +1336,7 @@ class AWSNovaSonicLLMService(LLMService):
             logger.debug(
                 "Wrapping assistant response trigger transcription with upstream UserStarted/StoppedSpeakingFrames"
             )
-            await self.push_frame(UserStartedSpeakingFrame(), direction=FrameDirection.UPSTREAM)
+            await self.broadcast_frame(UserStartedSpeakingFrame)
 
         # Send the transcription upstream for the user context aggregator
         frame = TranscriptionFrame(
@@ -1197,7 +1346,7 @@ class AWSNovaSonicLLMService(LLMService):
 
         # Finish wrapping the upstream transcription in UserStarted/StoppedSpeakingFrames if needed
         if should_wrap_in_user_started_stopped_speaking_frames:
-            await self.push_frame(UserStoppedSpeakingFrame(), direction=FrameDirection.UPSTREAM)
+            await self.broadcast_frame(UserStoppedSpeakingFrame)
 
         # Clear out the buffered user text
         self._user_text_buffer = ""
@@ -1262,7 +1411,7 @@ class AWSNovaSonicLLMService(LLMService):
         """
         if not self._is_assistant_response_trigger_needed():
             logger.warning(
-                f"Assistant response trigger not needed for model '{self._model}'; skipping. "
+                f"Assistant response trigger not needed for model '{self._settings.model}'; skipping. "
                 "An LLMRunFrame() should be sufficient to prompt the assistant to respond, "
                 "assuming the context ends in a user message."
             )
@@ -1290,9 +1439,9 @@ class AWSNovaSonicLLMService(LLMService):
             chunk_duration = 0.02  # what we might get from InputAudioRawFrame
             chunk_size = int(
                 chunk_duration
-                * self._params.input_sample_rate
-                * self._params.input_channel_count
-                * (self._params.input_sample_size / 8)
+                * self._audio_config.input_sample_rate
+                * self._audio_config.input_channel_count
+                * (self._audio_config.input_sample_size / 8)
             )  # e.g. 0.02 seconds of 16-bit (2-byte) PCM mono audio at 16kHz is 640 bytes
 
             # Lead with a bit of blank audio, if needed.

src/pipecat/services/aws/stt.py

@@ -10,12 +10,12 @@ This module provides a WebSocket-based connection to AWS Transcribe for real-tim
 speech-to-text transcription with support for multiple languages and audio formats.
 """
 
-import asyncio
 import json
 import os
 import random
 import string
-from typing import AsyncGenerator, Optional
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Optional
 
 from loguru import logger
 
@@ -29,6 +29,8 @@ from pipecat.frames.frames import (
     TranscriptionFrame,
 )
 from pipecat.services.aws.utils import build_event_message, decode_event, get_presigned_url
+from pipecat.services.settings import STTSettings
+from pipecat.services.stt_latency import AWS_TRANSCRIBE_TTFS_P99
 from pipecat.services.stt_service import WebsocketSTTService
 from pipecat.transcriptions.language import Language, resolve_language
 from pipecat.utils.time import time_now_iso8601
@@ -43,6 +45,13 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+@dataclass
+class AWSTranscribeSTTSettings(STTSettings):
+    """Settings for AWSTranscribeSTTService."""
+
+    pass
+
+
 class AWSTranscribeSTTService(WebsocketSTTService):
     """AWS Transcribe Speech-to-Text service using WebSocket streaming.
 
@@ -51,6 +60,9 @@ class AWSTranscribeSTTService(WebsocketSTTService):
     final transcription results.
     """
 
+    Settings = AWSTranscribeSTTSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
@@ -58,8 +70,10 @@ class AWSTranscribeSTTService(WebsocketSTTService):
         aws_access_key_id: Optional[str] = None,
         aws_session_token: Optional[str] = None,
         region: Optional[str] = None,
-        sample_rate: int = 16000,
-        language: Language = Language.EN,
+        sample_rate: Optional[int] = None,
+        language: Optional[Language] = None,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = AWS_TRANSCRIBE_TTFS_P99,
         **kwargs,
     ):
         """Initialize the AWS Transcribe STT service.
@@ -69,27 +83,49 @@ class AWSTranscribeSTTService(WebsocketSTTService):
             aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable.
             aws_session_token: AWS session token for temporary credentials. If None, uses AWS_SESSION_TOKEN environment variable.
             region: AWS region for the service.
-            sample_rate: Audio sample rate in Hz. Must be 8000 or 16000. Defaults to 16000.
-            language: Language for transcription. Defaults to English.
+            sample_rate: Audio sample rate in Hz. If None, uses the pipeline sample rate.
+                AWS Transcribe only supports 8000 or 16000 Hz; other values are
+                clamped to 16000 Hz at connect time.
+            language: Language for transcription.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSTranscribeSTTService.Settings(language=...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
             **kwargs: Additional arguments passed to parent STTService class.
         """
-        super().__init__(**kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            language=Language.EN,
+        )
 
-        self._settings = {
-            "sample_rate": sample_rate,
-            "language": language,
-            "media_encoding": "linear16",  # AWS expects raw PCM
-            "number_of_channels": 1,
-            "show_speaker_label": False,
-            "enable_channel_identification": False,
-        }
+        # 2. Apply direct init arg overrides (deprecated)
+        if language is not None:
+            self._warn_init_param_moved_to_settings("language", "language")
+            default_settings.language = language
 
-        # Validate sample rate - AWS Transcribe only supports 8000 Hz or 16000 Hz
-        if sample_rate not in [8000, 16000]:
-            logger.warning(
-                f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. Converting from {sample_rate} Hz to 16000 Hz."
-            )
-            self._settings["sample_rate"] = 16000
+        # 3. (No step 3, as there's no params object to apply)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            settings=default_settings,
+            **kwargs,
+        )
+
+        # Init-only connection config (not runtime-updatable).
+        self._media_encoding = "linear16"
+        self._number_of_channels = 1
+        self._show_speaker_label = False
+        self._enable_channel_identification = False
 
         self._credentials = {
             "aws_access_key_id": aws_access_key_id or os.getenv("AWS_ACCESS_KEY_ID"),
@@ -100,6 +136,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
 
         self._receive_task = None
 
+    def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as AWS Transcribe STT supports metrics generation.
+        """
+        return True
+
     def get_service_encoding(self, encoding: str) -> str:
         """Convert internal encoding format to AWS Transcribe format.
 
@@ -114,6 +158,16 @@ class AWSTranscribeSTTService(WebsocketSTTService):
         }
         return encoding_map.get(encoding, encoding)
 
+    async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
+        """Apply a settings delta and reconnect if anything changed."""
+        changed = await super()._update_settings(delta)
+
+        if changed and self._websocket:
+            await self._disconnect()
+            await self._connect()
+
+        return changed
+
     async def start(self, frame: StartFrame):
         """Initialize the connection when the service starts.
 
@@ -159,7 +213,6 @@ class AWSTranscribeSTTService(WebsocketSTTService):
                 await self._websocket.send(event_message)
                 # Start metrics after first chunk sent
                 await self.start_processing_metrics()
-                await self.start_ttfb_metrics()
             except Exception as e:
                 yield ErrorFrame(error=f"Error sending audio: {e}")
 
@@ -170,6 +223,8 @@ class AWSTranscribeSTTService(WebsocketSTTService):
 
         Establishes websocket connection and starts receive task.
         """
+        await super()._connect()
+
         await self._connect_websocket()
 
         if self._websocket and not self._receive_task:
@@ -180,6 +235,8 @@ class AWSTranscribeSTTService(WebsocketSTTService):
 
         Sends end-stream message and cleans up.
         """
+        await super()._disconnect()
+
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
@@ -202,9 +259,18 @@ class AWSTranscribeSTTService(WebsocketSTTService):
 
             logger.debug("Connecting to AWS Transcribe WebSocket")
 
-            language_code = self.language_to_service_language(Language(self._settings["language"]))
+            language_code = self._settings.language
             if not language_code:
-                raise ValueError(f"Unsupported language: {self._settings['language']}")
+                raise ValueError(f"Unsupported language: {language_code}")
+
+            # Validate sample rate — AWS Transcribe only supports 8000 or 16000 Hz
+            connect_sample_rate = self.sample_rate
+            if connect_sample_rate not in (8000, 16000):
+                logger.warning(
+                    f"AWS Transcribe only supports 8000 Hz or 16000 Hz sample rates. "
+                    f"Converting from {connect_sample_rate} Hz to 16000 Hz."
+                )
+                connect_sample_rate = 16000
 
             # Generate random websocket key
             websocket_key = "".join(
@@ -231,14 +297,14 @@ class AWSTranscribeSTTService(WebsocketSTTService):
                 },
                 language_code=language_code,
                 media_encoding=self.get_service_encoding(
-                    self._settings["media_encoding"]
+                    self._media_encoding
                 ),  # Convert to AWS format
-                sample_rate=self._settings["sample_rate"],
-                number_of_channels=self._settings["number_of_channels"],
+                sample_rate=connect_sample_rate,
+                number_of_channels=self._number_of_channels,
                 enable_partial_results_stabilization=True,
                 partial_results_stability="high",
-                show_speaker_label=self._settings["show_speaker_label"],
-                enable_channel_identification=self._settings["enable_channel_identification"],
+                show_speaker_label=self._show_speaker_label,
+                enable_channel_identification=self._enable_channel_identification,
             )
 
             logger.debug(f"{self} Connecting to WebSocket with URL: {presigned_url[:100]}...")
@@ -467,21 +533,20 @@ class AWSTranscribeSTTService(WebsocketSTTService):
                             is_final = not result.get("IsPartial", True)
 
                             if transcript:
-                                await self.stop_ttfb_metrics()
                                 if is_final:
                                     await self.push_frame(
                                         TranscriptionFrame(
                                             transcript,
                                             self._user_id,
                                             time_now_iso8601(),
-                                            self._settings["language"],
+                                            self._settings.language,
                                             result=result,
                                         )
                                     )
                                     await self._handle_transcription(
                                         transcript,
                                         is_final,
-                                        self._settings["language"],
+                                        self._settings.language,
                                     )
                                     await self.stop_processing_metrics()
                                 else:
@@ -490,7 +555,7 @@ class AWSTranscribeSTTService(WebsocketSTTService):
                                             transcript,
                                             self._user_id,
                                             time_now_iso8601(),
-                                            self._settings["language"],
+                                            self._settings.language,
                                             result=result,
                                         )
                                     )

src/pipecat/services/aws/tts.py

@@ -10,8 +10,8 @@ This module provides integration with Amazon Polly for text-to-speech synthesis,
 supporting multiple languages, voices, and SSML features.
 """
 
-import asyncio
 import os
+from dataclasses import dataclass, field
 from typing import AsyncGenerator, List, Optional
 
 from loguru import logger
@@ -22,9 +22,8 @@ from pipecat.frames.frames import (
     ErrorFrame,
     Frame,
     TTSAudioRawFrame,
-    TTSStartedFrame,
-    TTSStoppedFrame,
 )
+from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
 from pipecat.services.tts_service import TTSService
 from pipecat.transcriptions.language import Language, resolve_language
 from pipecat.utils.tracing.service_decorators import traced_tts
@@ -122,6 +121,25 @@ def language_to_aws_language(language: Language) -> Optional[str]:
     return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
 
 
+@dataclass
+class AWSPollyTTSSettings(TTSSettings):
+    """Settings for AWSPollyTTSService.
+
+    Parameters:
+        engine: TTS engine to use ('standard', 'neural', etc.).
+        pitch: Voice pitch adjustment (for standard engine only).
+        rate: Speech rate adjustment.
+        volume: Voice volume adjustment.
+        lexicon_names: List of pronunciation lexicons to apply.
+    """
+
+    engine: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    lexicon_names: List[str] | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
 class AWSPollyTTSService(TTSService):
     """AWS Polly text-to-speech service.
 
@@ -130,9 +148,15 @@ class AWSPollyTTSService(TTSService):
     options including prosody controls.
     """
 
+    Settings = AWSPollyTTSSettings
+    _settings: Settings
+
     class InputParams(BaseModel):
         """Input parameters for AWS Polly TTS configuration.
 
+        .. deprecated:: 0.0.105
+            Use ``AWSPollyTTSService.Settings`` directly via the ``settings`` parameter instead.
+
         Parameters:
             engine: TTS engine to use ('standard', 'neural', etc.).
             language: Language for synthesis. Defaults to English.
@@ -156,9 +180,10 @@ class AWSPollyTTSService(TTSService):
         aws_access_key_id: Optional[str] = None,
         aws_session_token: Optional[str] = None,
         region: Optional[str] = None,
-        voice_id: str = "Joanna",
+        voice_id: Optional[str] = None,
         sample_rate: Optional[int] = None,
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initializes the AWS Polly TTS service.
@@ -169,13 +194,59 @@ class AWSPollyTTSService(TTSService):
             aws_session_token: AWS session token for temporary credentials.
             region: AWS region for Polly service. Defaults to 'us-east-1'.
             voice_id: Voice ID to use for synthesis. Defaults to 'Joanna'.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSPollyTTSService.Settings(voice=...)`` instead.
+
             sample_rate: Audio sample rate. If None, uses service default.
             params: Additional input parameters for voice customization.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AWSPollyTTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional arguments passed to parent TTSService class.
         """
-        super().__init__(sample_rate=sample_rate, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            voice="Joanna",
+            language="en-US",
+            engine=None,
+            pitch=None,
+            rate=None,
+            volume=None,
+            lexicon_names=None,
+        )
 
-        params = params or AWSPollyTTSService.InputParams()
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice_id is not None:
+            self._warn_init_param_moved_to_settings("voice_id", "voice")
+            default_settings.voice = voice_id
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.engine = params.engine
+                default_settings.language = params.language if params.language else "en-US"
+                default_settings.pitch = params.pitch
+                default_settings.rate = params.rate
+                default_settings.volume = params.volume
+                default_settings.lexicon_names = params.lexicon_names
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            settings=default_settings,
+            **kwargs,
+        )
 
         # Get credentials from environment variables if not provided
         self._aws_params = {
@@ -186,21 +257,9 @@ class AWSPollyTTSService(TTSService):
         }
 
         self._aws_session = aioboto3.Session()
-        self._settings = {
-            "engine": params.engine,
-            "language": self.language_to_service_language(params.language)
-            if params.language
-            else "en-US",
-            "pitch": params.pitch,
-            "rate": params.rate,
-            "volume": params.volume,
-            "lexicon_names": params.lexicon_names,
-        }
 
         self._resampler = create_stream_resampler()
 
-        self.set_voice(voice_id)
-
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
 
@@ -223,19 +282,19 @@ class AWSPollyTTSService(TTSService):
     def _construct_ssml(self, text: str) -> str:
         ssml = "<speak>"
 
-        language = self._settings["language"]
+        language = self._settings.language
         ssml += f"<lang xml:lang='{language}'>"
 
         prosody_attrs = []
         # Prosody tags are only supported for standard and neural engines
-        if self._settings["engine"] == "standard":
-            if self._settings["pitch"]:
-                prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
+        if self._settings.engine == "standard":
+            if self._settings.pitch:
+                prosody_attrs.append(f"pitch='{self._settings.pitch}'")
 
-        if self._settings["rate"]:
-            prosody_attrs.append(f"rate='{self._settings['rate']}'")
-        if self._settings["volume"]:
-            prosody_attrs.append(f"volume='{self._settings['volume']}'")
+        if self._settings.rate:
+            prosody_attrs.append(f"rate='{self._settings.rate}'")
+        if self._settings.volume:
+            prosody_attrs.append(f"volume='{self._settings.volume}'")
 
         if prosody_attrs:
             ssml += f"<prosody {' '.join(prosody_attrs)}>"
@@ -254,11 +313,12 @@ class AWSPollyTTSService(TTSService):
         return ssml
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using AWS Polly.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the synthesized speech.
@@ -266,8 +326,6 @@ class AWSPollyTTSService(TTSService):
         logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
-            await self.start_ttfb_metrics()
-
             # Construct the parameters dictionary
             ssml = self._construct_ssml(text)
 
@@ -275,11 +333,11 @@ class AWSPollyTTSService(TTSService):
                 "Text": ssml,
                 "TextType": "ssml",
                 "OutputFormat": "pcm",
-                "VoiceId": self._voice_id,
-                "Engine": self._settings["engine"],
+                "VoiceId": self._settings.voice,
+                "Engine": self._settings.engine,
                 # AWS only supports 8000 and 16000 for PCM. We select 16000.
                 "SampleRate": "16000",
-                "LexiconNames": self._settings["lexicon_names"],
+                "LexiconNames": self._settings.lexicon_names,
             }
 
             # Filter out None values
@@ -299,25 +357,19 @@ class AWSPollyTTSService(TTSService):
 
                 await self.start_tts_usage_metrics(text)
 
-                yield TTSStartedFrame()
-
                 CHUNK_SIZE = self.chunk_size
 
                 for i in range(0, len(audio_data), CHUNK_SIZE):
                     chunk = audio_data[i : i + CHUNK_SIZE]
                     if len(chunk) > 0:
                         await self.stop_ttfb_metrics()
-                        frame = TTSAudioRawFrame(chunk, self.sample_rate, 1)
+                        frame = TTSAudioRawFrame(chunk, self.sample_rate, 1, context_id=context_id)
                         yield frame
 
-                yield TTSStoppedFrame()
         except (BotoCoreError, ClientError) as error:
             error_message = f"AWS Polly TTS error: {str(error)}"
             yield ErrorFrame(error=error_message)
 
-        finally:
-            yield TTSStoppedFrame()
-
 
 class PollyTTSService(AWSPollyTTSService):
     """Deprecated alias for AWSPollyTTSService.
@@ -327,6 +379,8 @@ class PollyTTSService(AWSPollyTTSService):
 
     """
 
+    Settings = AWSPollyTTSSettings
+
     def __init__(self, **kwargs):
         """Initialize the deprecated PollyTTSService.
 

src/pipecat/services/aws_nova_sonic/__init__.py

@@ -17,3 +17,8 @@ with warnings.catch_warnings():
         DeprecationWarning,
         stacklevel=2,
     )
+
+__all__ = [
+    "AWSNovaSonicLLMService",
+    "Params",
+]

src/pipecat/services/azure/common.py

@@ -8,8 +8,6 @@
 
 from typing import Optional
 
-from loguru import logger
-
 from pipecat.transcriptions.language import Language, resolve_language
 
 

src/pipecat/services/azure/image.py

@@ -12,14 +12,27 @@ using REST endpoints for creating images from text prompts.
 
 import asyncio
 import io
-from typing import AsyncGenerator
+from dataclasses import dataclass, field
+from typing import AsyncGenerator, Optional
 
 import aiohttp
-from loguru import logger
 from PIL import Image
 
 from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame
 from pipecat.services.image_service import ImageGenService
+from pipecat.services.settings import NOT_GIVEN, ImageGenSettings, _NotGiven
+
+
+@dataclass
+class AzureImageGenSettings(ImageGenSettings):
+    """Settings for the Azure image generation service.
+
+    Parameters:
+        model: Azure image generation model identifier.
+        image_size: Target size for generated images.
+    """
+
+    image_size: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
 
 
 class AzureImageGenServiceREST(ImageGenService):
@@ -30,33 +43,64 @@ class AzureImageGenServiceREST(ImageGenService):
     and automatic image download and processing.
     """
 
+    Settings = AzureImageGenSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
-        image_size: str,
+        image_size: Optional[str] = None,
         api_key: str,
         endpoint: str,
-        model: str,
+        model: Optional[str] = None,
         aiohttp_session: aiohttp.ClientSession,
         api_version="2023-06-01-preview",
+        settings: Optional[Settings] = None,
     ):
         """Initialize the AzureImageGenServiceREST.
 
         Args:
             image_size: Size specification for generated images (e.g., "1024x1024").
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureImageGenServiceREST.Settings(image_size=...)`` instead.
+
             api_key: Azure OpenAI API key for authentication.
             endpoint: Azure OpenAI endpoint URL.
             model: The image generation model to use.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureImageGenServiceREST.Settings(model=...)`` instead.
+
             aiohttp_session: Shared aiohttp session for HTTP requests.
             api_version: Azure API version string. Defaults to "2023-06-01-preview".
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
         """
-        super().__init__()
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            image_size=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        if image_size is not None:
+            self._warn_init_param_moved_to_settings("image_size", "image_size")
+            default_settings.image_size = image_size
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(settings=default_settings)
 
         self._api_key = api_key
         self._azure_endpoint = endpoint
         self._api_version = api_version
-        self.set_model_name(model)
-        self._image_size = image_size
         self._aiohttp_session = aiohttp_session
 
     async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
@@ -74,12 +118,13 @@ class AzureImageGenServiceREST(ImageGenService):
         headers = {"api-key": self._api_key, "Content-Type": "application/json"}
 
         body = {
-            # Enter your prompt text here
             "prompt": prompt,
-            "size": self._image_size,
             "n": 1,
         }
 
+        if self._settings.image_size is not None:
+            body["size"] = self._settings.image_size
+
         async with self._aiohttp_session.post(url, headers=headers, json=body) as submission:
             # We never get past this line, because this header isn't
             # defined on a 429 response, but something is eating our

src/pipecat/services/azure/llm.py

@@ -6,12 +6,23 @@
 
 """Azure OpenAI service implementation for the Pipecat AI framework."""
 
+from dataclasses import dataclass
+from typing import Optional
+
 from loguru import logger
 from openai import AsyncAzureOpenAI
 
+from pipecat.services.openai.base_llm import BaseOpenAILLMService
 from pipecat.services.openai.llm import OpenAILLMService
 
 
+@dataclass
+class AzureLLMSettings(BaseOpenAILLMService.Settings):
+    """Settings for AzureLLMService."""
+
+    pass
+
+
 class AzureLLMService(OpenAILLMService):
     """A service for interacting with Azure OpenAI using the OpenAI-compatible interface.
 
@@ -19,13 +30,16 @@ class AzureLLMService(OpenAILLMService):
     maintaining full compatibility with OpenAI's interface and functionality.
     """
 
+    Settings = AzureLLMSettings
+
     def __init__(
         self,
         *,
         api_key: str,
         endpoint: str,
-        model: str,
+        model: Optional[str] = None,
         api_version: str = "2024-09-01-preview",
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Azure LLM service.
@@ -33,15 +47,35 @@ class AzureLLMService(OpenAILLMService):
         Args:
             api_key: The API key for accessing Azure OpenAI.
             endpoint: The Azure endpoint URL.
-            model: The model identifier to use.
+            model: The model identifier to use. Defaults to "gpt-4.1".
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureLLMService.Settings(model=...)`` instead.
+
             api_version: Azure API version. Defaults to "2024-09-01-preview".
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional keyword arguments passed to OpenAILLMService.
         """
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(model="gpt-4.1")
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. (No step 3, as there's no params object to apply)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
         # Initialize variables before calling parent __init__() because that
         # will call create_client() and we need those values there.
         self._endpoint = endpoint
         self._api_version = api_version
-        super().__init__(api_key=api_key, model=model, **kwargs)
+        super().__init__(api_key=api_key, settings=default_settings, **kwargs)
 
     def create_client(self, api_key=None, base_url=None, **kwargs):
         """Create OpenAI-compatible client for Azure OpenAI endpoint.

src/pipecat/services/azure/realtime/llm.py

@@ -6,6 +6,8 @@
 
 """Azure OpenAI Realtime LLM service implementation."""
 
+from dataclasses import dataclass
+
 from loguru import logger
 
 from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService
@@ -18,6 +20,13 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+@dataclass
+class AzureRealtimeLLMSettings(OpenAIRealtimeLLMService.Settings):
+    """Settings for AzureRealtimeLLMService."""
+
+    pass
+
+
 class AzureRealtimeLLMService(OpenAIRealtimeLLMService):
     """Azure OpenAI Realtime LLM service with Azure-specific authentication.
 
@@ -26,6 +35,9 @@ class AzureRealtimeLLMService(OpenAIRealtimeLLMService):
     real-time audio and text communication capabilities as the base OpenAI service.
     """
 
+    Settings = AzureRealtimeLLMSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,

src/pipecat/services/azure/stt.py

@@ -11,7 +11,8 @@ Speech SDK for real-time audio transcription.
 """
 
 import asyncio
-from typing import AsyncGenerator, Optional
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Optional
 
 from loguru import logger
 
@@ -25,6 +26,8 @@ from pipecat.frames.frames import (
     TranscriptionFrame,
 )
 from pipecat.services.azure.common import language_to_azure_language
+from pipecat.services.settings import STTSettings
+from pipecat.services.stt_latency import AZURE_TTFS_P99
 from pipecat.services.stt_service import STTService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
@@ -32,6 +35,7 @@ from pipecat.utils.tracing.service_decorators import traced_stt
 
 try:
     from azure.cognitiveservices.speech import (
+        CancellationReason,
         ResultReason,
         SpeechConfig,
         SpeechRecognizer,
@@ -47,6 +51,13 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+@dataclass
+class AzureSTTSettings(STTSettings):
+    """Settings for AzureSTTService."""
+
+    pass
+
+
 class AzureSTTService(STTService):
     """Azure Speech-to-Text service for real-time audio transcription.
 
@@ -55,14 +66,20 @@ class AzureSTTService(STTService):
     provides real-time transcription results with timing information.
     """
 
+    Settings = AzureSTTSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
         api_key: str,
-        region: str,
-        language: Language = Language.EN_US,
+        region: Optional[str] = None,
+        language: Optional[Language] = Language.EN_US,
         sample_rate: Optional[int] = None,
+        private_endpoint: Optional[str] = None,
         endpoint_id: Optional[str] = None,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = AZURE_TTFS_P99,
         **kwargs,
     ):
         """Initialize the Azure STT service.
@@ -70,29 +87,75 @@ class AzureSTTService(STTService):
         Args:
             api_key: Azure Cognitive Services subscription key.
             region: Azure region for the Speech service (e.g., 'eastus').
+                Required unless ``private_endpoint`` is provided.
             language: Language for speech recognition. Defaults to English (US).
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureSTTService.Settings(language=...)`` instead.
+
             sample_rate: Audio sample rate in Hz. If None, uses service default.
+            private_endpoint: Private endpoint for STT behind firewall.
+                See https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-services-private-link?tabs=portal
             endpoint_id: Custom model endpoint id.
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
             **kwargs: Additional arguments passed to parent STTService.
         """
-        super().__init__(sample_rate=sample_rate, **kwargs)
-
-        self._speech_config = SpeechConfig(
-            subscription=api_key,
-            region=region,
-            speech_recognition_language=language_to_azure_language(language),
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            language=Language.EN_US,
         )
 
+        # 2. Apply direct init arg overrides (deprecated)
+        if language is not None and language != Language.EN_US:
+            self._warn_init_param_moved_to_settings("language", "language")
+            default_settings.language = language
+
+        # 3. (No step 3, as there's no params object to apply)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            settings=default_settings,
+            **kwargs,
+        )
+
+        recognition_language = default_settings.language or language_to_azure_language(
+            Language.EN_US
+        )
+
+        if not region and not private_endpoint:
+            raise ValueError("Either 'region' or 'private_endpoint' must be provided.")
+
+        if private_endpoint:
+            if region:
+                logger.warning(
+                    "Both 'region' and 'private_endpoint' provided; 'region' will be ignored."
+                )
+            self._speech_config = SpeechConfig(
+                subscription=api_key,
+                endpoint=private_endpoint,
+                speech_recognition_language=recognition_language,
+            )
+        else:
+            self._speech_config = SpeechConfig(
+                subscription=api_key,
+                region=region,
+                speech_recognition_language=recognition_language,
+            )
+
         if endpoint_id:
             self._speech_config.endpoint_id = endpoint_id
 
         self._audio_stream = None
         self._speech_recognizer = None
-        self._settings = {
-            "region": region,
-            "language": language_to_azure_language(language),
-            "sample_rate": sample_rate,
-        }
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate performance metrics.
@@ -102,6 +165,31 @@ class AzureSTTService(STTService):
         """
         return True
 
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert a Language enum to Azure service-specific language code.
+
+        Args:
+            language: The language to convert.
+
+        Returns:
+            The Azure-specific language identifier, or None if not supported.
+        """
+        return language_to_azure_language(language)
+
+    async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
+        """Apply a settings delta and reconnect if language changed."""
+        changed = await super()._update_settings(delta)
+
+        if "language" in changed:
+            self._speech_config.speech_recognition_language = (
+                self._settings.language or language_to_azure_language(Language.EN_US)
+            )
+            if self._audio_stream:
+                await self._disconnect()
+                await self._connect()
+
+        return changed
+
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
         """Process audio data for speech-to-text conversion.
 
@@ -116,7 +204,6 @@ class AzureSTTService(STTService):
         """
         try:
             await self.start_processing_metrics()
-            await self.start_ttfb_metrics()
             if self._audio_stream:
                 self._audio_stream.write(audio)
             yield None
@@ -126,14 +213,32 @@ class AzureSTTService(STTService):
     async def start(self, frame: StartFrame):
         """Start the speech recognition service.
 
-        Initializes the Azure speech recognizer with audio stream configuration
-        and begins continuous speech recognition.
-
         Args:
             frame: Frame indicating the start of processing.
         """
         await super().start(frame)
+        await self._connect()
 
+    async def stop(self, frame: EndFrame):
+        """Stop the speech recognition service.
+
+        Args:
+            frame: Frame indicating the end of processing.
+        """
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        """Cancel the speech recognition service.
+
+        Args:
+            frame: Frame indicating cancellation.
+        """
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def _connect(self):
+        """Initialize the Azure speech recognizer and begin continuous recognition."""
         if self._audio_stream:
             return
 
@@ -148,55 +253,33 @@ class AzureSTTService(STTService):
             )
             self._speech_recognizer.recognizing.connect(self._on_handle_recognizing)
             self._speech_recognizer.recognized.connect(self._on_handle_recognized)
+            self._speech_recognizer.canceled.connect(self._on_handle_canceled)
             self._speech_recognizer.start_continuous_recognition_async()
         except Exception as e:
             await self.push_error(
                 error_msg=f"Uncaught exception during initialization: {e}", exception=e
             )
 
-    async def stop(self, frame: EndFrame):
-        """Stop the speech recognition service.
-
-        Cleanly shuts down the Azure speech recognizer and closes audio streams.
-
-        Args:
-            frame: Frame indicating the end of processing.
-        """
-        await super().stop(frame)
-
-        if self._speech_recognizer:
-            self._speech_recognizer.stop_continuous_recognition_async()
-
-        if self._audio_stream:
-            self._audio_stream.close()
-
-    async def cancel(self, frame: CancelFrame):
-        """Cancel the speech recognition service.
-
-        Immediately stops recognition and closes resources.
-
-        Args:
-            frame: Frame indicating cancellation.
-        """
-        await super().cancel(frame)
-
+    async def _disconnect(self):
+        """Stop recognition and close audio streams."""
         if self._speech_recognizer:
             self._speech_recognizer.stop_continuous_recognition_async()
+            self._speech_recognizer = None
 
         if self._audio_stream:
             self._audio_stream.close()
+            self._audio_stream = None
 
     @traced_stt
     async def _handle_transcription(
         self, transcript: str, is_final: bool, language: Optional[Language] = None
     ):
         """Handle a transcription result with tracing."""
-        await self.stop_ttfb_metrics()
         await self.stop_processing_metrics()
 
     def _on_handle_recognized(self, event):
         if event.result.reason == ResultReason.RecognizedSpeech and len(event.result.text) > 0:
-            language = getattr(event.result, "language", None) or self._settings.get("language")
+            language = getattr(event.result, "language", None) or self._settings.language
             frame = TranscriptionFrame(
                 event.result.text,
                 self._user_id,
@@ -211,7 +294,7 @@ class AzureSTTService(STTService):
 
     def _on_handle_recognizing(self, event):
         if event.result.reason == ResultReason.RecognizingSpeech and len(event.result.text) > 0:
-            language = getattr(event.result, "language", None) or self._settings.get("language")
+            language = getattr(event.result, "language", None) or self._settings.language
             frame = InterimTranscriptionFrame(
                 event.result.text,
                 self._user_id,
@@ -220,3 +303,13 @@ class AzureSTTService(STTService):
                 result=event,
             )
             asyncio.run_coroutine_threadsafe(self.push_frame(frame), self.get_event_loop())
+
+    def _on_handle_canceled(self, event):
+        details = event.result.cancellation_details
+        if details.reason == CancellationReason.Error:
+            error_msg = f"Azure STT recognition canceled: {details.reason}"
+            if details.error_details:
+                error_msg += f" - {details.error_details}"
+            asyncio.run_coroutine_threadsafe(
+                self.push_error(error_msg=error_msg), self.get_event_loop()
+            )

src/pipecat/services/azure/tts.py

@@ -7,6 +7,7 @@
 """Azure Cognitive Services Text-to-Speech service implementations."""
 
 import asyncio
+from dataclasses import dataclass, field
 from typing import AsyncGenerator, Optional
 
 from loguru import logger
@@ -20,12 +21,12 @@ from pipecat.frames.frames import (
     InterruptionFrame,
     StartFrame,
     TTSAudioRawFrame,
-    TTSStartedFrame,
     TTSStoppedFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
 from pipecat.services.azure.common import language_to_azure_language
-from pipecat.services.tts_service import TTSService, WordTTSService
+from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
+from pipecat.services.tts_service import TextAggregationMode, TTSService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.tracing.service_decorators import traced_tts
 
@@ -65,6 +66,29 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma
     return sample_rate_map.get(sample_rate, SpeechSynthesisOutputFormat.Raw24Khz16BitMonoPcm)
 
 
+@dataclass
+class AzureTTSSettings(TTSSettings):
+    """Settings for AzureTTSService and AzureHttpTTSService.
+
+    Parameters:
+        emphasis: Emphasis level for speech ("strong", "moderate", "reduced").
+        pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high").
+        rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast").
+        role: Voice role for expression (e.g., "YoungAdultFemale").
+        style: Speaking style (e.g., "cheerful", "sad", "excited").
+        style_degree: Intensity of the speaking style (0.01 to 2.0).
+        volume: Volume level (e.g., "+20%", "loud", "x-soft").
+    """
+
+    emphasis: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    pitch: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    rate: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    role: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    style: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    style_degree: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    volume: str | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
 class AzureBaseTTSService:
     """Base mixin class for Azure Cognitive Services text-to-speech implementations.
 
@@ -73,6 +97,9 @@ class AzureBaseTTSService:
     This is a mixin class and should be used alongside TTSService or its subclasses.
     """
 
+    Settings = AzureTTSSettings
+    _settings: Settings
+
     # Define SSML escape mappings based on SSML reserved characters
     # See - https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-structure
     SSML_ESCAPE_CHARS = {
@@ -86,11 +113,14 @@ class AzureBaseTTSService:
     class InputParams(BaseModel):
         """Input parameters for Azure TTS voice configuration.
 
+        .. deprecated:: 0.0.105
+            Use ``settings=AzureBaseTTSService.Settings(...)`` instead.
+
         Parameters:
             emphasis: Emphasis level for speech ("strong", "moderate", "reduced").
             language: Language for synthesis. Defaults to English (US).
             pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high").
-            rate: Speech rate multiplier. Defaults to "1.05".
+            rate: Speech rate adjustment (e.g., "1.0", "1.25", "slow", "fast").
             role: Voice role for expression (e.g., "YoungAdultFemale").
             style: Speaking style (e.g., "cheerful", "sad", "excited").
             style_degree: Intensity of the speaking style (0.01 to 2.0).
@@ -100,7 +130,7 @@ class AzureBaseTTSService:
         emphasis: Optional[str] = None
         language: Optional[Language] = Language.EN_US
         pitch: Optional[str] = None
-        rate: Optional[str] = "1.05"
+        rate: Optional[str] = None
         role: Optional[str] = None
         style: Optional[str] = None
         style_degree: Optional[str] = None
@@ -111,8 +141,6 @@ class AzureBaseTTSService:
         *,
         api_key: str,
         region: str,
-        voice: str = "en-US-SaraNeural",
-        params: Optional[InputParams] = None,
     ):
         """Initialize Azure-specific configuration.
 
@@ -121,27 +149,9 @@ class AzureBaseTTSService:
         Args:
             api_key: Azure Cognitive Services subscription key.
             region: Azure region identifier (e.g., "eastus", "westus2").
-            voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
-            params: Voice and synthesis parameters configuration.
         """
-        params = params or AzureBaseTTSService.InputParams()
-
-        self._settings = {
-            "emphasis": params.emphasis,
-            "language": self.language_to_service_language(params.language)
-            if params.language
-            else "en-US",
-            "pitch": params.pitch,
-            "rate": params.rate,
-            "role": params.role,
-            "style": params.style,
-            "style_degree": params.style_degree,
-            "volume": params.volume,
-        }
-
         self._api_key = api_key
         self._region = region
-        self._voice_id = voice
         self._speech_synthesizer = None
 
     def language_to_service_language(self, language: Language) -> Optional[str]:
@@ -156,7 +166,7 @@ class AzureBaseTTSService:
         return language_to_azure_language(language)
 
     def _construct_ssml(self, text: str) -> str:
-        language = self._settings["language"]
+        language = self._settings.language
 
         # Escape special characters
         escaped_text = self._escape_text(text)
@@ -165,39 +175,42 @@ class AzureBaseTTSService:
             f"<speak version='1.0' xml:lang='{language}' "
             "xmlns='http://www.w3.org/2001/10/synthesis' "
             "xmlns:mstts='http://www.w3.org/2001/mstts'>"
-            f"<voice name='{self._voice_id}'>"
+            f"<voice name='{self._settings.voice}'>"
             "<mstts:silence type='Sentenceboundary' value='20ms' />"
         )
 
-        if self._settings["style"]:
-            ssml += f"<mstts:express-as style='{self._settings['style']}'"
-            if self._settings["style_degree"]:
-                ssml += f" styledegree='{self._settings['style_degree']}'"
-            if self._settings["role"]:
-                ssml += f" role='{self._settings['role']}'"
+        if self._settings.style:
+            ssml += f"<mstts:express-as style='{self._settings.style}'"
+            if self._settings.style_degree:
+                ssml += f" styledegree='{self._settings.style_degree}'"
+            if self._settings.role:
+                ssml += f" role='{self._settings.role}'"
             ssml += ">"
 
         prosody_attrs = []
-        if self._settings["rate"]:
-            prosody_attrs.append(f"rate='{self._settings['rate']}'")
-        if self._settings["pitch"]:
-            prosody_attrs.append(f"pitch='{self._settings['pitch']}'")
-        if self._settings["volume"]:
-            prosody_attrs.append(f"volume='{self._settings['volume']}'")
+        if self._settings.rate:
+            prosody_attrs.append(f"rate='{self._settings.rate}'")
+        if self._settings.pitch:
+            prosody_attrs.append(f"pitch='{self._settings.pitch}'")
+        if self._settings.volume:
+            prosody_attrs.append(f"volume='{self._settings.volume}'")
 
-        ssml += f"<prosody {' '.join(prosody_attrs)}>"
+        # Only wrap in prosody tag if there are prosody attributes
+        if prosody_attrs:
+            ssml += f"<prosody {' '.join(prosody_attrs)}>"
 
-        if self._settings["emphasis"]:
-            ssml += f"<emphasis level='{self._settings['emphasis']}'>"
+        if self._settings.emphasis:
+            ssml += f"<emphasis level='{self._settings.emphasis}'>"
 
         ssml += escaped_text
 
-        if self._settings["emphasis"]:
+        if self._settings.emphasis:
             ssml += "</emphasis>"
 
-        ssml += "</prosody>"
+        if prosody_attrs:
+            ssml += "</prosody>"
 
-        if self._settings["style"]:
+        if self._settings.style:
             ssml += "</mstts:express-as>"
 
         ssml += "</voice></speak>"
@@ -226,7 +239,7 @@ class AzureBaseTTSService:
         return escaped_text
 
 
-class AzureTTSService(WordTTSService, AzureBaseTTSService):
+class AzureTTSService(TTSService, AzureBaseTTSService):
     """Azure Cognitive Services streaming TTS service with word timestamps.
 
     Provides real-time text-to-speech synthesis using Azure's WebSocket-based
@@ -234,15 +247,19 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
     available for lower latency playback and accurate word-level synchronization.
     """
 
+    Settings = AzureTTSSettings
+
     def __init__(
         self,
         *,
         api_key: str,
         region: str,
-        voice: str = "en-US-SaraNeural",
+        voice: Optional[str] = None,
         sample_rate: Optional[int] = None,
         params: Optional[AzureBaseTTSService.InputParams] = None,
-        aggregate_sentences: bool = True,
+        settings: Optional[Settings] = None,
+        aggregate_sentences: Optional[bool] = None,
+        text_aggregation_mode: Optional[TextAggregationMode] = None,
         **kwargs,
     ):
         """Initialize the Azure streaming TTS service.
@@ -250,33 +267,94 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
         Args:
             api_key: Azure Cognitive Services subscription key.
             region: Azure region identifier (e.g., "eastus", "westus2").
-            voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
+            voice: Voice name to use for synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureTTSService.Settings(voice=...)`` instead.
+
             sample_rate: Audio sample rate in Hz. If None, uses service default.
             params: Voice and synthesis parameters configuration.
-            aggregate_sentences: Whether to aggregate sentences before synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureTTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            aggregate_sentences: Deprecated. Use text_aggregation_mode instead.
+
+                .. deprecated:: 0.0.104
+                    Use ``text_aggregation_mode`` instead.
+
+            text_aggregation_mode: How to aggregate text before synthesis.
             **kwargs: Additional arguments passed to parent WordTTSService.
         """
-        # Initialize WordTTSService first to set up word timestamp tracking
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            voice="en-US-SaraNeural",
+            language="en-US",
+            emphasis=None,
+            pitch=None,
+            rate=None,
+            role=None,
+            style=None,
+            style_degree=None,
+            volume=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice is not None:
+            self._warn_init_param_moved_to_settings("voice", "voice")
+            default_settings.voice = voice
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.emphasis = params.emphasis
+                default_settings.language = params.language if params.language else "en-US"
+                default_settings.pitch = params.pitch
+                default_settings.rate = params.rate
+                default_settings.role = params.role
+                default_settings.style = params.style
+                default_settings.style_degree = params.style_degree
+                default_settings.volume = params.volume
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
         super().__init__(
             aggregate_sentences=aggregate_sentences,
+            text_aggregation_mode=text_aggregation_mode,
             push_text_frames=False,  # We'll push text frames based on word timestamps
             push_stop_frames=True,
+            push_start_frame=True,
             pause_frame_processing=True,
             sample_rate=sample_rate,
+            settings=default_settings,
             **kwargs,
         )
 
         # Initialize Azure-specific functionality from mixin
-        self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
+        self._init_azure_base(api_key=api_key, region=region)
 
         self._speech_config = None
         self._speech_synthesizer = None
         self._audio_queue = asyncio.Queue()
         self._word_boundary_queue = asyncio.Queue()
         self._word_processor_task = None
-        self._started = False
-        self._first_chunk = True
         self._cumulative_audio_offset: float = 0.0  # Cumulative audio duration in seconds
+        self._current_sentence_base_offset: float = 0.0  # Base offset for current sentence
+        self._current_sentence_duration: float = 0.0  # Duration from Azure callback
+        self._current_sentence_max_word_offset: float = (
+            0.0  # Max word boundary offset seen in current sentence (for 8kHz workaround)
+        )
+        self._last_word: Optional[str] = None  # Track last word for punctuation merging
+        self._last_timestamp: Optional[float] = None  # Track last timestamp
+        self._current_context_id: Optional[str] = (
+            None  # Track current context_id for word timestamps
+        )
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
@@ -302,7 +380,7 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
             subscription=self._api_key,
             region=self._region,
         )
-        self._speech_config.speech_synthesis_language = self._settings["language"]
+        self._speech_config.speech_synthesis_language = self._settings.language
         self._speech_config.set_speech_synthesis_output_format(
             sample_rate_to_output_format(self.sample_rate)
         )
@@ -346,9 +424,34 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
         await self.cancel_task(self._word_processor_task)
         self._word_processor_task = None
 
+    def _is_cjk_language(self) -> bool:
+        """Check if the configured language is CJK (Chinese, Japanese, Korean).
+
+        Returns:
+            True if the language is CJK, False otherwise.
+        """
+        language = (self._settings.language if self._settings.language else "").lower()
+        # Check if language starts with CJK language codes
+        return language.startswith(("zh", "ja", "ko", "cmn", "yue", "wuu"))
+
+    def _is_punctuation_only(self, text: str) -> bool:
+        """Check if text consists only of punctuation and whitespace.
+
+        Args:
+            text: Text to check.
+
+        Returns:
+            True if text is only punctuation/whitespace, False otherwise.
+        """
+        return text and all(not c.isalnum() for c in text)
+
     def _handle_word_boundary(self, evt):
         """Handle word boundary events from Azure SDK.
 
+        Azure sends punctuation as separate word boundaries, and breaks CJK text
+        into individual characters/particles. This method routes to language-specific
+        handlers to properly merge and emit word boundaries.
+
         Args:
             evt: SpeechSynthesisWordBoundaryEventArgs from Azure Speech SDK
                 containing word text and audio offset timing.
@@ -359,23 +462,94 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
         word = evt.text
         sentence_relative_seconds = evt.audio_offset / 10_000_000.0
 
-        # Add cumulative offset to get absolute timestamp across sentences
-        absolute_seconds = self._cumulative_audio_offset + sentence_relative_seconds
+        # Use base offset captured at start of run_tts to avoid race conditions
+        # with callbacks from overlapping TTS requests
+        absolute_seconds = self._current_sentence_base_offset + sentence_relative_seconds
 
-        # Queue word timestamp for async processing
-        # Use thread-safe queue since this is called from Azure SDK thread
-        if word:
-            logger.trace(f"{self}: Word boundary - '{word}' at {absolute_seconds:.2f}s")
-            # Put in temporary queue - will be processed by async task
-            # Store as (word, timestamp_in_seconds) tuple
-            self._word_boundary_queue.put_nowait((word, absolute_seconds))
+        # Track max word offset for accurate cumulative timing
+        # (audio_duration from Azure doesn't always match word boundary offsets at 8kHz)
+        if sentence_relative_seconds > self._current_sentence_max_word_offset:
+            self._current_sentence_max_word_offset = sentence_relative_seconds
+
+        if not word:
+            return
+
+        # Route to language-specific handler
+        if self._is_cjk_language():
+            self._handle_cjk_word_boundary(word, absolute_seconds)
+        else:
+            self._handle_non_cjk_word_boundary(word, absolute_seconds)
+
+    def _emit_pending_word(self):
+        """Emit the currently buffered word if one exists."""
+        if self._last_word is not None:
+            self._word_boundary_queue.put_nowait((self._last_word, self._last_timestamp))
+            self._last_word = None
+            self._last_timestamp = None
+
+    def _handle_cjk_word_boundary(self, word: str, timestamp: float):
+        """Handle word boundaries for CJK languages (Chinese, Japanese, Korean).
+
+        CJK languages don't use spaces between words, so we merge characters together
+        and only emit at natural break points (punctuation or whitespace boundaries).
+        Without this logic, we don't get word output for CJK languages.
+
+        Args:
+            word: The word/character from Azure.
+            timestamp: Timestamp in seconds.
+        """
+        # First word: just store it
+        if self._last_word is None:
+            self._last_word = word
+            self._last_timestamp = timestamp
+            return
+
+        # Punctuation: merge and emit (natural break)
+        if self._is_punctuation_only(word):
+            self._last_word += word
+            self._emit_pending_word()
+            return
+
+        # Whitespace: emit before boundary, start new segment
+        if word.strip() != word:
+            self._emit_pending_word()
+            self._last_word = word
+            self._last_timestamp = timestamp
+            return
+
+        # Default: continue merging CJK characters
+        self._last_word += word
+
+    def _handle_non_cjk_word_boundary(self, word: str, timestamp: float):
+        """Handle word boundaries for non-CJK languages.
+
+        Non-CJK languages use spaces between words, so we emit each word separately
+        after merging any trailing punctuation.
+
+        Args:
+            word: The word from Azure.
+            timestamp: Timestamp in seconds.
+        """
+        # Punctuation: merge with previous word (don't emit yet)
+        if self._is_punctuation_only(word) and self._last_word is not None:
+            self._last_word += word
+            return
+
+        # Regular word: emit previous, store current
+        if self._last_word is not None:
+            self._word_boundary_queue.put_nowait((self._last_word, self._last_timestamp))
+        self._last_word = word
+        self._last_timestamp = timestamp
 
     async def _word_processor_task_handler(self):
         """Process word timestamps from the queue and call add_word_timestamps."""
         while True:
             try:
                 word, timestamp_seconds = await self._word_boundary_queue.get()
-                await self.add_word_timestamps([(word, timestamp_seconds)])
+                if self._current_context_id:
+                    await self.add_word_timestamps(
+                        [(word, timestamp_seconds)], self._current_context_id
+                    )
                 self._word_boundary_queue.task_done()
             except asyncio.CancelledError:
                 break
@@ -397,9 +571,15 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
         Args:
             evt: Completion event from Azure Speech SDK.
         """
-        # Update cumulative audio offset for next sentence
+        # Flush any pending word before completing
+        if self._last_word is not None:
+            self._word_boundary_queue.put_nowait((self._last_word, self._last_timestamp))
+            self._last_word = None
+            self._last_timestamp = None
+
+        # Store duration for cumulative offset calculation
         if evt.result and evt.result.audio_duration:
-            self._cumulative_audio_offset += evt.result.audio_duration.total_seconds()
+            self._current_sentence_duration = evt.result.audio_duration.total_seconds()
 
         self._audio_queue.put_nowait(None)  # Signal completion
 
@@ -413,9 +593,13 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
         # User cancellation (from interruption) is expected, not an error
         if reason == CancellationReason.CancelledByUser:
             logger.debug(f"{self}: Speech synthesis canceled by user (interruption)")
+            self._audio_queue.put_nowait(None)
         else:
-            logger.warning(f"{self}: Speech synthesis canceled: {reason}")
-        self._audio_queue.put_nowait(None)
+            details = evt.result.cancellation_details
+            error_msg = f"Azure TTS synthesis canceled: {reason}"
+            if details.error_details:
+                error_msg += f" - {details.error_details}"
+            self._audio_queue.put_nowait(Exception(error_msg))
 
     async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
         """Push a frame and handle state changes.
@@ -427,16 +611,20 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
         await super().push_frame(frame, direction)
         if isinstance(frame, (TTSStoppedFrame, InterruptionFrame)):
             self._reset_state()
-            if isinstance(frame, TTSStoppedFrame):
-                await self.add_word_timestamps([("Reset", 0)])
+            if isinstance(frame, TTSStoppedFrame) and self._current_context_id:
+                await self.add_word_timestamps([("Reset", 0)], self._current_context_id)
 
     def _reset_state(self):
         """Reset TTS state between turns."""
-        self._started = False
-        self._first_chunk = True
         self._cumulative_audio_offset = 0.0
+        self._current_sentence_base_offset = 0.0
+        self._current_sentence_duration = 0.0
+        self._current_sentence_max_word_offset = 0.0
+        self._last_word = None
+        self._last_timestamp = None
+        self._current_context_id = None
 
-    async def flush_audio(self):
+    async def flush_audio(self, context_id: Optional[str] = None):
         """Flush any pending audio data."""
         logger.trace(f"{self}: flushing audio")
 
@@ -478,11 +666,12 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
                 break
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Azure's streaming synthesis.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing synthesized speech data.
@@ -501,11 +690,13 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
                 return
 
             try:
-                if not self._started:
-                    await self.start_ttfb_metrics()
-                    yield TTSStartedFrame()
-                    self._started = True
-                    self._first_chunk = True
+                self._current_context_id = context_id
+
+                # Capture base offset BEFORE starting synthesis to avoid race conditions
+                # Word boundary callbacks will use this value
+                self._current_sentence_base_offset = self._cumulative_audio_offset
+                self._current_sentence_duration = 0.0
+                self._current_sentence_max_word_offset = 0.0
 
                 ssml = self._construct_ssml(text)
                 self._speech_synthesizer.speak_ssml_async(ssml)
@@ -517,22 +708,31 @@ class AzureTTSService(WordTTSService, AzureBaseTTSService):
                     chunk = await self._audio_queue.get()
                     if chunk is None:  # End of stream
                         break
-
-                    if self._first_chunk:
-                        await self.stop_ttfb_metrics()
-                        await self.start_word_timestamps()
-                        self._first_chunk = False
+                    if isinstance(chunk, Exception):  # Error from _handle_canceled
+                        yield ErrorFrame(error=str(chunk))
+                        break
 
                     frame = TTSAudioRawFrame(
                         audio=chunk,
                         sample_rate=self.sample_rate,
                         num_channels=1,
+                        context_id=context_id,
                     )
                     yield frame
 
+                # Update cumulative offset for next sentence
+                # At 8kHz, Azure's audio_duration doesn't match word boundary offsets,
+                # so we use max_word_offset as a workaround. At other sample rates,
+                # audio_duration is accurate.
+                # TODO: Remove after Azure fixes word boundary timing at 8kHz
+                if self.sample_rate == 8000:
+                    self._cumulative_audio_offset += self._current_sentence_max_word_offset
+                else:
+                    self._cumulative_audio_offset += self._current_sentence_duration
+
             except Exception as e:
                 yield ErrorFrame(error=f"Unknown error occurred: {e}")
-                yield TTSStoppedFrame()
+                yield TTSStoppedFrame(context_id=context_id)
                 self._reset_state()
                 return
 
@@ -548,14 +748,17 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
     required and simpler integration is preferred.
     """
 
+    Settings = AzureTTSSettings
+
     def __init__(
         self,
         *,
         api_key: str,
         region: str,
-        voice: str = "en-US-SaraNeural",
+        voice: Optional[str] = None,
         sample_rate: Optional[int] = None,
         params: Optional[AzureBaseTTSService.InputParams] = None,
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Azure HTTP TTS service.
@@ -563,15 +766,67 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
         Args:
             api_key: Azure Cognitive Services subscription key.
             region: Azure region identifier (e.g., "eastus", "westus2").
-            voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
+            voice: Voice name to use for synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureHttpTTSService.Settings(voice=...)`` instead.
+
             sample_rate: Audio sample rate in Hz. If None, uses service default.
             params: Voice and synthesis parameters configuration.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=AzureHttpTTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional arguments passed to parent TTSService.
         """
-        super().__init__(sample_rate=sample_rate, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            voice="en-US-SaraNeural",
+            language="en-US",
+            emphasis=None,
+            pitch=None,
+            rate=None,
+            role=None,
+            style=None,
+            style_degree=None,
+            volume=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice is not None:
+            self._warn_init_param_moved_to_settings("voice", "voice")
+            default_settings.voice = voice
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.emphasis = params.emphasis
+                default_settings.language = params.language if params.language else "en-US"
+                default_settings.pitch = params.pitch
+                default_settings.rate = params.rate
+                default_settings.role = params.role
+                default_settings.style = params.style
+                default_settings.style_degree = params.style_degree
+                default_settings.volume = params.volume
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            settings=default_settings,
+            **kwargs,
+        )
 
         # Initialize Azure-specific functionality from mixin
-        self._init_azure_base(api_key=api_key, region=region, voice=voice, params=params)
+        self._init_azure_base(api_key=api_key, region=region)
 
         self._speech_config = None
         self._speech_synthesizer = None
@@ -599,7 +854,7 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
             subscription=self._api_key,
             region=self._region,
         )
-        self._speech_config.speech_synthesis_language = self._settings["language"]
+        self._speech_config.speech_synthesis_language = self._settings.language
         self._speech_config.set_speech_synthesis_output_format(
             sample_rate_to_output_format(self.sample_rate)
         )
@@ -608,19 +863,18 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
         )
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Azure's HTTP synthesis API.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the complete synthesized speech.
         """
         logger.debug(f"{self}: Generating TTS [{text}]")
 
-        await self.start_ttfb_metrics()
-
         ssml = self._construct_ssml(text)
 
         result = await asyncio.to_thread(self._speech_synthesizer.speak_ssml, ssml)
@@ -628,14 +882,13 @@ class AzureHttpTTSService(TTSService, AzureBaseTTSService):
         if result.reason == ResultReason.SynthesizingAudioCompleted:
             await self.start_tts_usage_metrics(text)
             await self.stop_ttfb_metrics()
-            yield TTSStartedFrame()
             # Azure always sends a 44-byte header. Strip it off.
             yield TTSAudioRawFrame(
                 audio=result.audio_data[44:],
                 sample_rate=self.sample_rate,
                 num_channels=1,
+                context_id=context_id,
             )
-            yield TTSStoppedFrame()
         elif result.reason == ResultReason.Canceled:
             cancellation_details = result.cancellation_details
             logger.warning(f"Speech synthesis canceled: {cancellation_details.reason}")

src/pipecat/services/camb/__init__.py

@@ -0,0 +1,5 @@
+#
+# Copyright (c) 2024–2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#

src/pipecat/services/camb/tts.py

@@ -0,0 +1,392 @@
+#
+# Copyright (c) 2024–2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Camb.ai MARS text-to-speech service implementation.
+
+This module provides TTS functionality using Camb.ai's MARS model family,
+offering high-quality text-to-speech synthesis with streaming support.
+
+Features:
+    - MARS models: mars-flash (fast), mars-pro (high quality)
+    - 140+ languages supported
+    - Real-time streaming via official SDK
+    - Model-specific sample rates: mars-pro (48kHz), mars-flash (22.05kHz)
+"""
+
+from dataclasses import dataclass, field
+from typing import Any, AsyncGenerator, Dict, Optional
+
+from camb import StreamTtsOutputConfiguration
+from camb.client import AsyncCambAI
+from loguru import logger
+from pydantic import BaseModel, Field
+
+from pipecat.frames.frames import (
+    ErrorFrame,
+    Frame,
+    StartFrame,
+    TTSAudioRawFrame,
+)
+from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
+from pipecat.services.tts_service import TTSService
+from pipecat.transcriptions.language import Language, resolve_language
+from pipecat.utils.tracing.service_decorators import traced_tts
+
+# Model-specific sample rates
+MODEL_SAMPLE_RATES: Dict[str, int] = {
+    "mars-flash": 22050,  # 22.05kHz
+    "mars-pro": 48000,  # 48kHz
+    "mars-instruct": 22050,  # 22.05kHz
+}
+
+
+def language_to_camb_language(language: Language) -> Optional[str]:
+    """Convert a Pipecat Language enum to Camb.ai language code.
+
+    Args:
+        language: The Language enum value to convert.
+
+    Returns:
+        The corresponding Camb.ai language code (BCP-47 format), or None if not supported.
+    """
+    LANGUAGE_MAP = {
+        Language.EN: "en-us",
+        Language.EN_US: "en-us",
+        Language.EN_GB: "en-gb",
+        Language.EN_AU: "en-au",
+        Language.ES: "es-es",
+        Language.ES_ES: "es-es",
+        Language.ES_MX: "es-mx",
+        Language.FR: "fr-fr",
+        Language.FR_FR: "fr-fr",
+        Language.FR_CA: "fr-ca",
+        Language.DE: "de-de",
+        Language.DE_DE: "de-de",
+        Language.IT: "it-it",
+        Language.PT: "pt-pt",
+        Language.PT_BR: "pt-br",
+        Language.PT_PT: "pt-pt",
+        Language.NL: "nl-nl",
+        Language.PL: "pl-pl",
+        Language.RU: "ru-ru",
+        Language.JA: "ja-jp",
+        Language.KO: "ko-kr",
+        Language.ZH: "zh-cn",
+        Language.ZH_CN: "zh-cn",
+        Language.ZH_TW: "zh-tw",
+        Language.AR: "ar-sa",
+        Language.HI: "hi-in",
+        Language.TR: "tr-tr",
+        Language.VI: "vi-vn",
+        Language.TH: "th-th",
+        Language.ID: "id-id",
+        Language.MS: "ms-my",
+        Language.SV: "sv-se",
+        Language.DA: "da-dk",
+        Language.NO: "no-no",
+        Language.FI: "fi-fi",
+        Language.CS: "cs-cz",
+        Language.EL: "el-gr",
+        Language.HE: "he-il",
+        Language.HU: "hu-hu",
+        Language.RO: "ro-ro",
+        Language.SK: "sk-sk",
+        Language.UK: "uk-ua",
+        Language.BG: "bg-bg",
+        Language.HR: "hr-hr",
+        Language.SR: "sr-rs",
+        Language.SL: "sl-si",
+        Language.CA: "ca-es",
+        Language.EU: "eu-es",
+        Language.GL: "gl-es",
+        Language.AF: "af-za",
+        Language.SW: "sw-ke",
+        Language.TA: "ta-in",
+        Language.TE: "te-in",
+        Language.BN: "bn-in",
+        Language.MR: "mr-in",
+        Language.GU: "gu-in",
+        Language.KN: "kn-in",
+        Language.ML: "ml-in",
+        Language.PA: "pa-in",
+        Language.UR: "ur-pk",
+        Language.FA: "fa-ir",
+        Language.TL: "tl-ph",
+    }
+
+    return resolve_language(language, LANGUAGE_MAP, use_base_code=True)
+
+
+def _get_aligned_audio(buffer: bytes) -> tuple[bytes, bytes]:
+    """Split buffer into aligned audio (2-byte samples) and remainder.
+
+    Args:
+        buffer: Raw audio bytes to align.
+
+    Returns:
+        Tuple of (aligned audio bytes, remaining bytes).
+    """
+    aligned_size = (len(buffer) // 2) * 2
+    return buffer[:aligned_size], buffer[aligned_size:]
+
+
+@dataclass
+class CambTTSSettings(TTSSettings):
+    """Settings for CambTTSService.
+
+    Parameters:
+        voice: Camb.ai voice ID. Overrides ``TTSSettings.voice`` (str) because
+            Camb.ai uses integer voice IDs.
+        user_instructions: Custom instructions for mars-instruct model only.
+            Ignored for other models. Max 1000 characters.
+    """
+
+    voice: int | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    user_instructions: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
+class CambTTSService(TTSService):
+    """Camb.ai MARS text-to-speech service using the official SDK.
+
+    Converts text to speech using Camb.ai's MARS TTS models with support for
+    multiple languages.
+
+    Models:
+        - mars-flash: Fast inference, 22.05kHz output (default)
+        - mars-pro: High quality, 48kHz output
+
+    Example::
+
+        # Basic usage with mars-flash (fast)
+        tts = CambTTSService(
+            api_key="your-api-key",
+            settings=CambTTSService.Settings(
+                model="mars-flash"
+            )
+        )
+
+        # High quality with mars-pro
+        tts = CambTTSService(
+            api_key="your-api-key",
+            settings=CambTTSService.Settings(
+                voice=12345,
+                model="mars-pro",
+            )
+        )
+    """
+
+    Settings = CambTTSSettings
+    _settings: Settings
+
+    class InputParams(BaseModel):
+        """Input parameters for Camb.ai TTS configuration.
+
+        .. deprecated:: 0.0.105
+            Use ``settings=CambTTSService.Settings(...)`` instead.
+
+        Parameters:
+            language: Language for synthesis (BCP-47 format). Defaults to English.
+            user_instructions: Custom instructions for mars-instruct model only.
+                Ignored for other models. Max 1000 characters.
+        """
+
+        language: Optional[Language] = Language.EN
+        user_instructions: Optional[str] = Field(
+            default=None,
+            max_length=1000,
+            description="Custom instructions for mars-instruct model only. "
+            "Use to control tone, style, or pronunciation. Max 1000 characters.",
+        )
+
+    def __init__(
+        self,
+        *,
+        api_key: str,
+        voice_id: Optional[int] = None,
+        model: Optional[str] = None,
+        timeout: float = 60.0,
+        sample_rate: Optional[int] = None,
+        params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
+        **kwargs,
+    ):
+        """Initialize the Camb.ai TTS service.
+
+        Args:
+            api_key: Camb.ai API key for authentication.
+            voice_id: Voice ID to use.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CambTTSService.Settings(voice=...)`` instead.
+
+            model: TTS model to use. Options: "mars-flash" (fast), "mars-pro" (high quality).
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CambTTSService.Settings(model=...)`` instead.
+
+            timeout: Request timeout in seconds. Defaults to 60.0 (minimum recommended
+                by Camb.ai).
+            sample_rate: Audio sample rate in Hz. If None, uses model-specific default.
+            params: Additional voice parameters. If None, uses defaults.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CambTTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            **kwargs: Additional arguments passed to parent TTSService.
+        """
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="mars-flash",
+            voice=147320,
+            language="en-us",
+            user_instructions=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+        if voice_id is not None:
+            self._warn_init_param_moved_to_settings("voice_id", "voice")
+            default_settings.voice = voice_id
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                if params.language is not None:
+                    default_settings.language = params.language
+                if params.user_instructions is not None:
+                    default_settings.user_instructions = params.user_instructions
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        # Warn if sample rate doesn't match model's supported rate
+        _model = default_settings.model
+        if sample_rate and sample_rate != MODEL_SAMPLE_RATES.get(_model):
+            logger.warning(
+                f"Camb.ai's {_model} model only supports {MODEL_SAMPLE_RATES.get(_model)}Hz "
+                f"sample rate. Current rate of {sample_rate}Hz may cause issues."
+            )
+
+        super().__init__(
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            settings=default_settings,
+            **kwargs,
+        )
+
+        self._api_key = api_key
+        self._timeout = timeout
+
+        self._client = None
+
+    def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Camb.ai service supports metrics generation.
+        """
+        return True
+
+    def language_to_service_language(self, language: Language) -> Optional[str]:
+        """Convert a Language enum to Camb.ai language format.
+
+        Args:
+            language: The language to convert.
+
+        Returns:
+            The Camb.ai-specific language code, or None if not supported.
+        """
+        return language_to_camb_language(language)
+
+    async def start(self, frame: StartFrame):
+        """Start the Camb.ai TTS service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
+        await super().start(frame)
+
+        self._client = AsyncCambAI(api_key=self._api_key, timeout=self._timeout)
+
+        # Use model-specific sample rate if not explicitly specified
+        if not self._init_sample_rate:
+            self._sample_rate = MODEL_SAMPLE_RATES.get(self._settings.model, 22050)
+
+    @traced_tts
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
+        """Generate speech from text using Camb.ai's TTS API.
+
+        Args:
+            text: The text to synthesize into speech (max 3000 characters).
+            context_id: The context ID for tracking audio frames.
+
+        Yields:
+            Frame: Audio frames containing the synthesized speech.
+        """
+        logger.debug(f"{self}: Generating TTS [{text}]")
+
+        # Validate text length
+        if len(text) > 3000:
+            logger.warning("Text too long for Camb.ai TTS (max 3000 chars), truncating")
+            text = text[:3000]
+
+        try:
+            # Build SDK parameters
+            tts_kwargs: Dict[str, Any] = {
+                "text": text,
+                "voice_id": self._settings.voice,
+                "language": self._settings.language,
+                "speech_model": self._settings.model,
+                "output_configuration": StreamTtsOutputConfiguration(format="pcm_s16le"),
+            }
+
+            # Add user instructions if using mars-instruct model
+            if self._settings.model == "mars-instruct" and self._settings.user_instructions:
+                tts_kwargs["user_instructions"] = self._settings.user_instructions
+
+            await self.start_tts_usage_metrics(text)
+
+            assert self._client is not None, "Camb.ai TTS service not initialized"
+
+            # Buffer for aligning chunks to 2-byte boundaries (16-bit PCM)
+            audio_buffer = b""
+
+            # Stream audio chunks from SDK
+            async for chunk in self._client.text_to_speech.tts(**tts_kwargs):
+                if chunk:
+                    await self.stop_ttfb_metrics()
+                    audio_buffer += chunk
+
+                    # Only yield complete 16-bit samples (2 bytes per sample)
+                    aligned_audio, audio_buffer = _get_aligned_audio(audio_buffer)
+                    if aligned_audio:
+                        yield TTSAudioRawFrame(
+                            audio=aligned_audio,
+                            sample_rate=self.sample_rate,
+                            num_channels=1,
+                            context_id=context_id,
+                        )
+
+            # Yield any remaining complete samples
+            if len(audio_buffer) >= 2:
+                aligned_audio, _ = _get_aligned_audio(audio_buffer)
+                if aligned_audio:
+                    yield TTSAudioRawFrame(
+                        audio=aligned_audio,
+                        sample_rate=self.sample_rate,
+                        num_channels=1,
+                        context_id=context_id,
+                    )
+
+        except Exception as e:
+            yield ErrorFrame(error=f"Camb.ai TTS error: {e}")

src/pipecat/services/cartesia/stt.py

@@ -12,7 +12,8 @@ the Cartesia Live transcription API for real-time speech recognition.
 
 import json
 import urllib.parse
-from typing import AsyncGenerator, Optional
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Optional
 
 from loguru import logger
 
@@ -27,6 +28,8 @@ from pipecat.frames.frames import (
     VADUserStoppedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.settings import STTSettings
+from pipecat.services.stt_latency import CARTESIA_TTFS_P99
 from pipecat.services.stt_service import WebsocketSTTService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
@@ -41,11 +44,19 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+@dataclass
+class CartesiaSTTSettings(STTSettings):
+    """Settings for CartesiaSTTService."""
+
+    pass
+
+
 class CartesiaLiveOptions:
     """Configuration options for Cartesia Live STT service.
 
-    Manages transcription parameters including model selection, language,
-    audio encoding format, and sample rate settings.
+    .. deprecated:: 0.0.105
+        Use ``settings=CartesiaSTTService.Settings(...)`` for model/language and
+        direct ``__init__`` parameters for encoding/sample_rate instead.
     """
 
     def __init__(
@@ -128,15 +139,26 @@ class CartesiaSTTService(WebsocketSTTService):
     Provides real-time speech transcription through WebSocket connection
     to Cartesia's Live transcription service. Supports both interim and
     final transcriptions with configurable models and languages.
+
+    Cartesia disconnects WebSocket connections after 3 minutes of inactivity.
+    The timeout resets with each message (audio data or text command) sent to
+    the server. Silence-based keepalive is enabled by default to prevent this.
+    See: https://docs.cartesia.ai/api-reference/stt/stt
     """
 
+    Settings = CartesiaSTTSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
         api_key: str,
         base_url: str = "",
-        sample_rate: int = 16000,
+        encoding: str = "pcm_s16le",
+        sample_rate: Optional[int] = None,
         live_options: Optional[CartesiaLiveOptions] = None,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = CARTESIA_TTFS_P99,
         **kwargs,
     ):
         """Initialize CartesiaSTTService with API key and options.
@@ -144,34 +166,61 @@ class CartesiaSTTService(WebsocketSTTService):
         Args:
             api_key: Authentication key for Cartesia API.
             base_url: Custom API endpoint URL. If empty, uses default.
-            sample_rate: Audio sample rate in Hz. Defaults to 16000.
+            encoding: Audio encoding format. Defaults to "pcm_s16le".
+            sample_rate: Audio sample rate in Hz. If None, uses the pipeline
+                sample rate.
             live_options: Configuration options for transcription service.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CartesiaSTTService.Settings(...)`` for model/language
+                    and direct init parameters for encoding/sample_rate instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
             **kwargs: Additional arguments passed to parent STTService.
         """
-        sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
-        super().__init__(sample_rate=sample_rate, **kwargs)
-
-        default_options = CartesiaLiveOptions(
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
             model="ink-whisper",
             language=Language.EN.value,
-            encoding="pcm_s16le",
-            sample_rate=sample_rate,
         )
 
-        merged_options = default_options.to_dict()
-        if live_options:
-            merged_options.update(live_options.to_dict())
-            # Filter out "None" string values
-            merged_options = {
-                k: v for k, v in merged_options.items() if not isinstance(v, str) or v != "None"
-            }
+        # 2. Apply live_options overrides — only if settings not provided
+        if live_options is not None:
+            self._warn_init_param_moved_to_settings("live_options")
+            if not settings:
+                if live_options.sample_rate and sample_rate is None:
+                    sample_rate = live_options.sample_rate
+                if live_options.encoding:
+                    encoding = live_options.encoding
+                if live_options.model:
+                    default_settings.model = live_options.model
+                if live_options.language:
+                    lang = live_options.language
+                    default_settings.language = lang.value if isinstance(lang, Language) else lang
+
+        # 3. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            keepalive_timeout=120,
+            keepalive_interval=30,
+            settings=default_settings,
+            **kwargs,
+        )
 
-        self._settings = merged_options
-        self.set_model_name(merged_options["model"])
         self._api_key = api_key
         self._base_url = base_url or "api.cartesia.ai"
         self._receive_task = None
 
+        # Init-only audio config (not runtime-updatable).
+        self._encoding = encoding
+
     def can_generate_metrics(self) -> bool:
         """Check if the service can generate processing metrics.
 
@@ -207,9 +256,8 @@ class CartesiaSTTService(WebsocketSTTService):
         await super().cancel(frame)
         await self._disconnect()
 
-    async def start_metrics(self):
+    async def _start_metrics(self):
         """Start performance metrics collection for transcription processing."""
-        await self.start_ttfb_metrics()
         await self.start_processing_metrics()
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -222,7 +270,7 @@ class CartesiaSTTService(WebsocketSTTService):
         await super().process_frame(frame, direction)
 
         if isinstance(frame, VADUserStartedSpeakingFrame):
-            await self.start_metrics()
+            await self._start_metrics()
         elif isinstance(frame, VADUserStoppedSpeakingFrame):
             # Send finalize command to flush the transcription session
             if self._websocket and self._websocket.state is State.OPEN:
@@ -247,23 +295,53 @@ class CartesiaSTTService(WebsocketSTTService):
     async def _connect(self):
         await self._connect_websocket()
 
+        await super()._connect()
+
         if self._websocket and not self._receive_task:
             self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
 
     async def _disconnect(self):
+        await super()._disconnect()
+
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
 
         await self._disconnect_websocket()
 
+    async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
+        """Apply a settings delta.
+
+        Args:
+            delta: A :class:`STTSettings` (or ``CartesiaSTTService.Settings``) delta.
+
+        Returns:
+            Dict mapping changed field names to their previous values.
+        """
+        changed = await super()._update_settings(delta)
+
+        # TODO: someday we could reconnect here to apply updated settings.
+        # Code might look something like the below:
+        # if changed:
+        #     await self._disconnect()
+        #     await self._connect()
+
+        self._warn_unhandled_updated_settings(changed)
+
+        return changed
+
     async def _connect_websocket(self):
         try:
             if self._websocket and self._websocket.state is State.OPEN:
                 return
             logger.debug("Connecting to Cartesia STT")
 
-            params = self._settings
+            params = {
+                "model": self._settings.model,
+                "language": self._settings.language,
+                "encoding": self._encoding,
+                "sample_rate": str(self.sample_rate),
+            }
             ws_url = f"wss://{self._base_url}/stt/websocket?{urllib.parse.urlencode(params)}"
             headers = {"Cartesia-Version": "2025-04-16", "X-API-Key": self._api_key}
 
@@ -288,7 +366,7 @@ class CartesiaSTTService(WebsocketSTTService):
             return self._websocket
         raise Exception("Websocket not connected")
 
-    async def _process_messages(self):
+    async def _receive_messages(self):
         """Process incoming WebSocket messages."""
         async for message in self._get_websocket():
             try:
@@ -299,14 +377,6 @@ class CartesiaSTTService(WebsocketSTTService):
             except Exception as e:
                 logger.error(f"Error processing message: {e}")
 
-    async def _receive_messages(self):
-        while True:
-            await self._process_messages()
-            # Cartesia times out after 5 minutes of innactivity (no keepalive
-            # mechanism is available). So, we try to reconnect.
-            logger.debug(f"{self} Cartesia connection was disconnected (timeout?), reconnecting")
-            await self._connect_websocket()
-
     async def _process_response(self, data):
         if "type" in data:
             if data["type"] == "transcript":
@@ -338,7 +408,6 @@ class CartesiaSTTService(WebsocketSTTService):
                 pass
 
         if len(transcript) > 0:
-            await self.stop_ttfb_metrics()
             if is_final:
                 await self.push_frame(
                     TranscriptionFrame(

src/pipecat/services/cartesia/tts.py

@@ -8,39 +8,32 @@
 
 import base64
 import json
-import uuid
-import warnings
+from dataclasses import dataclass, field
 from enum import Enum
-from typing import AsyncGenerator, List, Literal, Optional
+from typing import AsyncGenerator, List, Optional
 
+import aiohttp
 from loguru import logger
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
 from pipecat.frames.frames import (
     CancelFrame,
     EndFrame,
     ErrorFrame,
     Frame,
-    InterruptionFrame,
     StartFrame,
     TTSAudioRawFrame,
-    TTSStartedFrame,
     TTSStoppedFrame,
 )
-from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.tts_service import AudioContextWordTTSService, TTSService
+from pipecat.services.settings import NOT_GIVEN, TTSSettings, _NotGiven
+from pipecat.services.tts_service import TextAggregationMode, TTSService, WebsocketTTSService
 from pipecat.transcriptions.language import Language, resolve_language
 from pipecat.utils.text.base_text_aggregator import BaseTextAggregator
 from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator
 from pipecat.utils.tracing.service_decorators import traced_tts
 
-# Suppress regex warnings from pydub (used by cartesia)
-warnings.filterwarnings("ignore", message="invalid escape sequence", category=SyntaxWarning)
-
-
 # See .env.example for Cartesia configuration needed
 try:
-    from cartesia import AsyncCartesia
     from websockets.asyncio.client import connect as websocket_connect
     from websockets.protocol import State
 except ModuleNotFoundError as e:
@@ -192,33 +185,45 @@ class CartesiaEmotion(str, Enum):
     DETERMINED = "determined"
 
 
-class CartesiaTTSService(AudioContextWordTTSService):
+@dataclass
+class CartesiaTTSSettings(TTSSettings):
+    """Settings for CartesiaTTSService and CartesiaHttpTTSService.
+
+    Parameters:
+        generation_config: Generation configuration for Sonic-3 models. Includes volume,
+            speed (numeric), and emotion (string) parameters.
+        pronunciation_dict_id: The ID of the pronunciation dictionary to use for
+            custom pronunciations.
+    """
+
+    generation_config: GenerationConfig | None | _NotGiven = field(
+        default_factory=lambda: NOT_GIVEN
+    )
+    pronunciation_dict_id: str | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
+class CartesiaTTSService(WebsocketTTSService):
     """Cartesia TTS service with WebSocket streaming and word timestamps.
 
     Provides text-to-speech using Cartesia's streaming WebSocket API.
     Supports word-level timestamps, audio context management, and various voice
-    customization options including speed and emotion controls.
+    customization options including generation configuration.
     """
 
+    Settings = CartesiaTTSSettings
+    _settings: Settings
+
     class InputParams(BaseModel):
         """Input parameters for Cartesia TTS configuration.
 
         Parameters:
             language: Language to use for synthesis.
-            speed: Voice speed control for non-Sonic-3 models (literal values).
-            emotion: List of emotion controls for non-Sonic-3 models.
-
-                .. deprecated:: 0.0.68
-                        The `emotion` parameter is deprecated and will be removed in a future version.
-
             generation_config: Generation configuration for Sonic-3 models. Includes volume,
                 speed (numeric), and emotion (string) parameters.
             pronunciation_dict_id: The ID of the pronunciation dictionary to use for custom pronunciations.
         """
 
         language: Optional[Language] = Language.EN
-        speed: Optional[Literal["slow", "normal", "fast"]] = None
-        emotion: Optional[List[str]] = []
         generation_config: Optional[GenerationConfig] = None
         pronunciation_dict_id: Optional[str] = None
 
@@ -226,16 +231,18 @@ class CartesiaTTSService(AudioContextWordTTSService):
         self,
         *,
         api_key: str,
-        voice_id: str,
+        voice_id: Optional[str] = None,
         cartesia_version: str = "2025-04-16",
         url: str = "wss://api.cartesia.ai/tts/websocket",
-        model: str = "sonic-3",
+        model: Optional[str] = None,
         sample_rate: Optional[int] = None,
         encoding: str = "pcm_s16le",
         container: str = "raw",
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
         text_aggregator: Optional[BaseTextAggregator] = None,
-        aggregate_sentences: Optional[bool] = True,
+        text_aggregation_mode: Optional[TextAggregationMode] = None,
+        aggregate_sentences: Optional[bool] = None,
         **kwargs,
     ):
         """Initialize the Cartesia TTS service.
@@ -243,37 +250,95 @@ class CartesiaTTSService(AudioContextWordTTSService):
         Args:
             api_key: Cartesia API key for authentication.
             voice_id: ID of the voice to use for synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CartesiaTTSService.Settings(voice=...)`` instead.
+
             cartesia_version: API version string for Cartesia service.
             url: WebSocket URL for Cartesia TTS API.
             model: TTS model to use (e.g., "sonic-3").
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CartesiaTTSService.Settings(model=...)`` instead.
+
             sample_rate: Audio sample rate. If None, uses default.
             encoding: Audio encoding format.
             container: Audio container format.
             params: Additional input parameters for voice customization.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CartesiaTTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             text_aggregator: Custom text aggregator for processing input text.
 
                 .. deprecated:: 0.0.95
                     Use an LLMTextProcessor before the TTSService for custom text aggregation.
 
+            text_aggregation_mode: How to aggregate incoming text before synthesis.
             aggregate_sentences: Whether to aggregate sentences within the TTSService.
+
+                .. deprecated:: 0.0.104
+                    Use ``text_aggregation_mode`` instead.
+
             **kwargs: Additional arguments passed to the parent service.
         """
-        # Aggregating sentences still gives cleaner-sounding results and fewer
-        # artifacts than streaming one word at a time. On average, waiting for a
-        # full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
-        # 3 model, and it's worth it for the better audio quality.
+        # By default, we aggregate sentences before sending to TTS. This adds
+        # ~200-300ms of latency per sentence (waiting for the sentence-ending
+        # punctuation token from the LLM). Setting
+        # text_aggregation_mode=TextAggregationMode.TOKEN streams tokens
+        # directly, which reduces latency. Streaming quality is good but less
+        # tested than sentence aggregation.
+        # TODO: Consider making TOKEN the default for Cartesia in 1.0.
         #
         # We also don't want to automatically push LLM response text frames,
         # because the context aggregators will add them to the LLM context even
         # if we're interrupted. Cartesia gives us word-by-word timestamps. We
         # can use those to generate text frames ourselves aligned with the
         # playout timing of the audio!
+
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="sonic-3",
+            voice=None,
+            language=Language.EN,
+            generation_config=None,
+            pronunciation_dict_id=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice_id is not None:
+            self._warn_init_param_moved_to_settings("voice_id", "voice")
+            default_settings.voice = voice_id
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                if params.language is not None:
+                    default_settings.language = params.language
+                if params.generation_config is not None:
+                    default_settings.generation_config = params.generation_config
+                if params.pronunciation_dict_id is not None:
+                    default_settings.pronunciation_dict_id = params.pronunciation_dict_id
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
         super().__init__(
+            text_aggregation_mode=text_aggregation_mode,
             aggregate_sentences=aggregate_sentences,
             push_text_frames=False,
-            pause_frame_processing=True,
+            pause_frame_processing=False,
             sample_rate=sample_rate,
+            push_start_frame=True,
             text_aggregator=text_aggregator,
+            settings=default_settings,
             **kwargs,
         )
 
@@ -283,31 +348,19 @@ class CartesiaTTSService(AudioContextWordTTSService):
             #    The preferred way of taking advantage of Cartesia SSML Tags is
             #    to use an LLMTextProcessor and/or a text_transformer to identify
             #    and insert these tags for the purpose of the TTS service alone.
-            self._text_aggregator = SkipTagsAggregator([("<spell>", "</spell>")])
-
-        params = params or CartesiaTTSService.InputParams()
+            self._text_aggregator = SkipTagsAggregator(
+                [("<spell>", "</spell>")], aggregation_type=self._text_aggregation_mode
+            )
 
         self._api_key = api_key
         self._cartesia_version = cartesia_version
         self._url = url
-        self._settings = {
-            "output_format": {
-                "container": container,
-                "encoding": encoding,
-                "sample_rate": 0,
-            },
-            "language": self.language_to_service_language(params.language)
-            if params.language
-            else None,
-            "speed": params.speed,
-            "emotion": params.emotion,
-            "generation_config": params.generation_config,
-            "pronunciation_dict_id": params.pronunciation_dict_id,
-        }
-        self.set_model_name(model)
-        self.set_voice(voice_id)
 
-        self._context_id = None
+        # Audio output format — init-only, not runtime-updatable
+        self._output_container = container
+        self._output_encoding = encoding
+        self._output_sample_rate = 0  # Set in start() from self.sample_rate
+
         self._receive_task = None
 
     def can_generate_metrics(self) -> bool:
@@ -318,16 +371,6 @@ class CartesiaTTSService(AudioContextWordTTSService):
         """
         return True
 
-    async def set_model(self, model: str):
-        """Set the TTS model.
-
-        Args:
-            model: The model name to use for synthesis.
-        """
-        self._model_id = model
-        await super().set_model(model)
-        logger.info(f"Switching TTS model to: [{model}]")
-
     def language_to_service_language(self, language: Language) -> Optional[str]:
         """Convert a Language enum to Cartesia language format.
 
@@ -392,7 +435,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
         Returns:
             List of (word, start_time) tuples processed for the language.
         """
-        current_language = self._settings.get("language")
+        current_language = self._settings.language
 
         # Check if this is a CJK language (if language is None, treat as non-CJK)
         if current_language and self._is_cjk_language(current_language):
@@ -409,48 +452,41 @@ class CartesiaTTSService(AudioContextWordTTSService):
             return list(zip(words, starts))
 
     def _build_msg(
-        self, text: str = "", continue_transcript: bool = True, add_timestamps: bool = True
+        self,
+        text: str = "",
+        continue_transcript: bool = True,
+        add_timestamps: bool = True,
+        context_id: str = "",
     ):
         voice_config = {}
         voice_config["mode"] = "id"
-        voice_config["id"] = self._voice_id
-
-        if self._settings["emotion"]:
-            with warnings.catch_warnings():
-                warnings.simplefilter("always")
-                warnings.warn(
-                    "The 'emotion' parameter in __experimental_controls is deprecated and will be removed in a future version.",
-                    DeprecationWarning,
-                    stacklevel=2,
-                )
-            voice_config["__experimental_controls"] = {}
-            if self._settings["emotion"]:
-                voice_config["__experimental_controls"]["emotion"] = self._settings["emotion"]
+        voice_config["id"] = self._settings.voice
 
         msg = {
             "transcript": text,
             "continue": continue_transcript,
-            "context_id": self._context_id,
-            "model_id": self.model_name,
+            "context_id": context_id,
+            "model_id": self._settings.model,
             "voice": voice_config,
-            "output_format": self._settings["output_format"],
+            "output_format": {
+                "container": self._output_container,
+                "encoding": self._output_encoding,
+                "sample_rate": self._output_sample_rate,
+            },
             "add_timestamps": add_timestamps,
-            "use_original_timestamps": False if self.model_name == "sonic" else True,
+            "use_original_timestamps": False if self._settings.model == "sonic" else True,
         }
 
-        if self._settings["language"]:
-            msg["language"] = self._settings["language"]
+        if self._settings.language:
+            msg["language"] = self._settings.language
 
-        if self._settings["speed"]:
-            msg["speed"] = self._settings["speed"]
-
-        if self._settings["generation_config"]:
-            msg["generation_config"] = self._settings["generation_config"].model_dump(
+        if self._settings.generation_config:
+            msg["generation_config"] = self._settings.generation_config.model_dump(
                 exclude_none=True
             )
 
-        if self._settings["pronunciation_dict_id"]:
-            msg["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
+        if self._settings.pronunciation_dict_id:
+            msg["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
 
         return json.dumps(msg)
 
@@ -461,7 +497,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
             frame: The start frame containing initialization parameters.
         """
         await super().start(frame)
-        self._settings["output_format"]["sample_rate"] = self.sample_rate
+        self._output_sample_rate = self.sample_rate
         await self._connect()
 
     async def stop(self, frame: EndFrame):
@@ -483,12 +519,16 @@ class CartesiaTTSService(AudioContextWordTTSService):
         await self._disconnect()
 
     async def _connect(self):
+        await super()._connect()
+
         await self._connect_websocket()
 
         if self._websocket and not self._receive_task:
             self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
 
     async def _disconnect(self):
+        await super()._disconnect()
+
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
@@ -519,7 +559,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
         except Exception as e:
             await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
         finally:
-            self._context_id = None
+            await self.remove_active_audio_context()
             self._websocket = None
             await self._call_event_handler("on_disconnected")
 
@@ -528,52 +568,65 @@ class CartesiaTTSService(AudioContextWordTTSService):
             return self._websocket
         raise Exception("Websocket not connected")
 
-    async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
-        await super()._handle_interruption(frame, direction)
+    async def on_audio_context_interrupted(self, context_id: str):
+        """Cancel the active Cartesia context when the bot is interrupted."""
         await self.stop_all_metrics()
-        if self._context_id:
-            cancel_msg = json.dumps({"context_id": self._context_id, "cancel": True})
+        if context_id:
+            cancel_msg = json.dumps({"context_id": context_id, "cancel": True})
             await self._get_websocket().send(cancel_msg)
-            self._context_id = None
 
-    async def flush_audio(self):
-        """Flush any pending audio and finalize the current context."""
-        if not self._context_id or not self._websocket:
+    async def on_audio_context_completed(self, context_id: str):
+        """Close the Cartesia context after all audio has been played.
+
+        No close message is needed: the server already considers the context
+        done once it has sent its ``done`` message, which is handled in
+        ``_process_messages``.
+        """
+        pass
+
+    async def flush_audio(self, context_id: Optional[str] = None):
+        """Flush any pending audio and finalize the current context.
+
+        Args:
+            context_id: The specific context to flush. If None, falls back to the
+                currently active context.
+        """
+        flush_id = context_id or self.get_active_audio_context_id()
+        if not flush_id or not self._websocket:
             return
         logger.trace(f"{self}: flushing audio")
-        msg = self._build_msg(text="", continue_transcript=False)
+        msg = self._build_msg(text="", continue_transcript=False, context_id=flush_id)
         await self._websocket.send(msg)
-        self._context_id = None
 
     async def _process_messages(self):
         async for message in self._get_websocket():
             msg = json.loads(message)
             if not msg or not self.audio_context_available(msg["context_id"]):
                 continue
+            ctx_id = msg["context_id"]
             if msg["type"] == "done":
                 await self.stop_ttfb_metrics()
-                await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)])
-                await self.remove_audio_context(msg["context_id"])
+                await self.add_word_timestamps([("TTSStoppedFrame", 0), ("Reset", 0)], ctx_id)
+                await self.remove_audio_context(ctx_id)
             elif msg["type"] == "timestamps":
                 # Process the timestamps based on language before adding them
                 processed_timestamps = self._process_word_timestamps_for_language(
                     msg["word_timestamps"]["words"], msg["word_timestamps"]["start"]
                 )
-                await self.add_word_timestamps(processed_timestamps)
+                await self.add_word_timestamps(processed_timestamps, ctx_id)
             elif msg["type"] == "chunk":
-                await self.stop_ttfb_metrics()
-                await self.start_word_timestamps()
                 frame = TTSAudioRawFrame(
                     audio=base64.b64decode(msg["data"]),
                     sample_rate=self.sample_rate,
                     num_channels=1,
+                    context_id=ctx_id,
                 )
-                await self.append_to_audio_context(msg["context_id"], frame)
+                await self.append_to_audio_context(ctx_id, frame)
             elif msg["type"] == "error":
-                await self.push_frame(TTSStoppedFrame())
+                await self.push_frame(TTSStoppedFrame(context_id=ctx_id))
                 await self.stop_all_metrics()
                 await self.push_error(error_msg=f"Error: {msg}")
-                self._context_id = None
+                self.reset_active_audio_context()
             else:
                 await self.push_error(error_msg=f"Error, unknown message type: {msg}")
 
@@ -586,35 +639,33 @@ class CartesiaTTSService(AudioContextWordTTSService):
             await self._connect_websocket()
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Cartesia's streaming API.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the synthesized speech.
         """
-        logger.debug(f"{self}: Generating TTS [{text}]")
+        if not self._is_streaming_tokens:
+            logger.debug(f"{self}: Generating TTS [{text}]")
+        else:
+            logger.trace(f"{self}: Generating TTS [{text}]")
 
         try:
             if not self._websocket or self._websocket.state is State.CLOSED:
                 await self._connect()
 
-            if not self._context_id:
-                await self.start_ttfb_metrics()
-                yield TTSStartedFrame()
-                self._context_id = str(uuid.uuid4())
-                await self.create_audio_context(self._context_id)
-
-            msg = self._build_msg(text=text)
+            msg = self._build_msg(text=text, context_id=context_id)
 
             try:
                 await self._get_websocket().send(msg)
                 await self.start_tts_usage_metrics(text)
             except Exception as e:
                 yield ErrorFrame(error=f"Unknown error occurred: {e}")
-                yield TTSStoppedFrame()
+                yield TTSStoppedFrame(context_id=context_id)
                 await self._disconnect()
                 await self._connect()
                 return
@@ -631,25 +682,20 @@ class CartesiaHttpTTSService(TTSService):
     integration is preferred.
     """
 
+    Settings = CartesiaTTSSettings
+    _settings: Settings
+
     class InputParams(BaseModel):
         """Input parameters for Cartesia HTTP TTS configuration.
 
         Parameters:
             language: Language to use for synthesis.
-            speed: Voice speed control for non-Sonic-3 models (literal values).
-            emotion: List of emotion controls for non-Sonic-3 models.
-
-                .. deprecated:: 0.0.68
-                        The `emotion` parameter is deprecated and will be removed in a future version.
-
             generation_config: Generation configuration for Sonic-3 models. Includes volume,
                 speed (numeric), and emotion (string) parameters.
             pronunciation_dict_id: The ID of the pronunciation dictionary to use for custom pronunciations.
         """
 
         language: Optional[Language] = Language.EN
-        speed: Optional[Literal["slow", "normal", "fast"]] = None
-        emotion: Optional[List[str]] = Field(default_factory=list)
         generation_config: Optional[GenerationConfig] = None
         pronunciation_dict_id: Optional[str] = None
 
@@ -657,14 +703,16 @@ class CartesiaHttpTTSService(TTSService):
         self,
         *,
         api_key: str,
-        voice_id: str,
-        model: str = "sonic-3",
+        voice_id: Optional[str] = None,
+        model: Optional[str] = None,
         base_url: str = "https://api.cartesia.ai",
-        cartesia_version: str = "2024-11-13",
+        cartesia_version: str = "2026-03-01",
+        aiohttp_session: Optional[aiohttp.ClientSession] = None,
         sample_rate: Optional[int] = None,
         encoding: str = "pcm_s16le",
         container: str = "raw",
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Cartesia HTTP TTS service.
@@ -672,43 +720,82 @@ class CartesiaHttpTTSService(TTSService):
         Args:
             api_key: Cartesia API key for authentication.
             voice_id: ID of the voice to use for synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CartesiaHttpTTSService.Settings(voice=...)`` instead.
+
             model: TTS model to use (e.g., "sonic-3").
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CartesiaHttpTTSService.Settings(model=...)`` instead.
+
             base_url: Base URL for Cartesia HTTP API.
             cartesia_version: API version string for Cartesia service.
+            aiohttp_session: Optional aiohttp ClientSession for HTTP requests.
+                If not provided, a session will be created and managed internally.
             sample_rate: Audio sample rate. If None, uses default.
             encoding: Audio encoding format.
             container: Audio container format.
             params: Additional input parameters for voice customization.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CartesiaHttpTTSService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional arguments passed to the parent TTSService.
         """
-        super().__init__(sample_rate=sample_rate, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="sonic-3",
+            voice=None,
+            language=Language.EN,
+            generation_config=None,
+            pronunciation_dict_id=None,
+        )
 
-        params = params or CartesiaHttpTTSService.InputParams()
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice_id is not None:
+            self._warn_init_param_moved_to_settings("voice_id", "voice")
+            default_settings.voice = voice_id
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                if params.language is not None:
+                    default_settings.language = params.language
+                if params.generation_config is not None:
+                    default_settings.generation_config = params.generation_config
+                if params.pronunciation_dict_id is not None:
+                    default_settings.pronunciation_dict_id = params.pronunciation_dict_id
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            settings=default_settings,
+            **kwargs,
+        )
 
         self._api_key = api_key
         self._base_url = base_url
         self._cartesia_version = cartesia_version
-        self._settings = {
-            "output_format": {
-                "container": container,
-                "encoding": encoding,
-                "sample_rate": 0,
-            },
-            "language": self.language_to_service_language(params.language)
-            if params.language
-            else None,
-            "speed": params.speed,
-            "emotion": params.emotion,
-            "generation_config": params.generation_config,
-            "pronunciation_dict_id": params.pronunciation_dict_id,
-        }
-        self.set_voice(voice_id)
-        self.set_model_name(model)
 
-        self._client = AsyncCartesia(
-            api_key=api_key,
-            base_url=base_url,
-        )
+        # Audio output format — init-only, not runtime-updatable
+        self._output_container = container
+        self._output_encoding = encoding
+        self._output_sample_rate = 0  # Set in start() from self.sample_rate
+
+        self._session: aiohttp.ClientSession | None = aiohttp_session
+        self._owns_session = aiohttp_session is None
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
@@ -736,7 +823,15 @@ class CartesiaHttpTTSService(TTSService):
             frame: The start frame containing initialization parameters.
         """
         await super().start(frame)
-        self._settings["output_format"]["sample_rate"] = self.sample_rate
+        self._output_sample_rate = self.sample_rate
+        if self._owns_session:
+            self._session = aiohttp.ClientSession()
+
+    async def _close_session(self):
+        """Close the HTTP session if we own it."""
+        if self._owns_session and self._session:
+            await self._session.close()
+            self._session = None
 
     async def stop(self, frame: EndFrame):
         """Stop the Cartesia HTTP TTS service.
@@ -745,7 +840,7 @@ class CartesiaHttpTTSService(TTSService):
             frame: The end frame.
         """
         await super().stop(frame)
-        await self._client.close()
+        await self._close_session()
 
     async def cancel(self, frame: CancelFrame):
         """Cancel the Cartesia HTTP TTS service.
@@ -754,14 +849,15 @@ class CartesiaHttpTTSService(TTSService):
             frame: The cancel frame.
         """
         await super().cancel(frame)
-        await self._client.close()
+        await self._close_session()
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Cartesia's HTTP API.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the synthesized speech.
@@ -769,44 +865,31 @@ class CartesiaHttpTTSService(TTSService):
         logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
-            voice_config = {"mode": "id", "id": self._voice_id}
+            voice_config = {"mode": "id", "id": self._settings.voice}
 
-            if self._settings["emotion"]:
-                with warnings.catch_warnings():
-                    warnings.simplefilter("always")
-                    warnings.warn(
-                        "The 'emotion' parameter in voice.__experimental_controls is deprecated and will be removed in a future version.",
-                        DeprecationWarning,
-                        stacklevel=2,
-                    )
-                voice_config["__experimental_controls"] = {"emotion": self._settings["emotion"]}
-
-            await self.start_ttfb_metrics()
-
-            payload = {
-                "model_id": self._model_name,
-                "transcript": text,
-                "voice": voice_config,
-                "output_format": self._settings["output_format"],
+            output_format = {
+                "container": self._output_container,
+                "encoding": self._output_encoding,
+                "sample_rate": self._output_sample_rate,
             }
 
-            if self._settings["language"]:
-                payload["language"] = self._settings["language"]
+            payload = {
+                "model_id": self._settings.model,
+                "transcript": text,
+                "voice": voice_config,
+                "output_format": output_format,
+            }
 
-            if self._settings["speed"]:
-                payload["speed"] = self._settings["speed"]
+            if self._settings.language:
+                payload["language"] = self._settings.language
 
-            if self._settings["generation_config"]:
-                payload["generation_config"] = self._settings["generation_config"].model_dump(
+            if self._settings.generation_config:
+                payload["generation_config"] = self._settings.generation_config.model_dump(
                     exclude_none=True
                 )
 
-            if self._settings["pronunciation_dict_id"]:
-                payload["pronunciation_dict_id"] = self._settings["pronunciation_dict_id"]
-
-            yield TTSStartedFrame()
-
-            session = await self._client._get_session()
+            if self._settings.pronunciation_dict_id:
+                payload["pronunciation_dict_id"] = self._settings.pronunciation_dict_id
 
             headers = {
                 "Cartesia-Version": self._cartesia_version,
@@ -816,7 +899,7 @@ class CartesiaHttpTTSService(TTSService):
 
             url = f"{self._base_url}/tts/bytes"
 
-            async with session.post(url, json=payload, headers=headers) as response:
+            async with self._session.post(url, json=payload, headers=headers) as response:
                 if response.status != 200:
                     error_text = await response.text()
                     yield ErrorFrame(error=f"Cartesia API error: {error_text}")
@@ -830,6 +913,7 @@ class CartesiaHttpTTSService(TTSService):
                 audio=audio_data,
                 sample_rate=self.sample_rate,
                 num_channels=1,
+                context_id=context_id,
             )
 
             yield frame
@@ -838,4 +922,3 @@ class CartesiaHttpTTSService(TTSService):
             yield ErrorFrame(error=f"Unknown error occurred: {e}")
         finally:
             await self.stop_ttfb_metrics()
-            yield TTSStoppedFrame()

src/pipecat/services/cerebras/llm.py

@@ -6,14 +6,23 @@
 
 """Cerebras LLM service implementation using OpenAI-compatible interface."""
 
-from typing import List
+from dataclasses import dataclass
+from typing import Optional
 
 from loguru import logger
 
 from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams
+from pipecat.services.openai.base_llm import BaseOpenAILLMService
 from pipecat.services.openai.llm import OpenAILLMService
 
 
+@dataclass
+class CerebrasLLMSettings(BaseOpenAILLMService.Settings):
+    """Settings for CerebrasLLMService."""
+
+    pass
+
+
 class CerebrasLLMService(OpenAILLMService):
     """A service for interacting with Cerebras's API using the OpenAI-compatible interface.
 
@@ -21,12 +30,16 @@ class CerebrasLLMService(OpenAILLMService):
     maintaining full compatibility with OpenAI's interface and functionality.
     """
 
+    Settings = CerebrasLLMSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
         api_key: str,
         base_url: str = "https://api.cerebras.ai/v1",
-        model: str = "gpt-oss-120b",
+        model: Optional[str] = None,
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Cerebras LLM service.
@@ -35,9 +48,29 @@ class CerebrasLLMService(OpenAILLMService):
             api_key: The API key for accessing Cerebras's API.
             base_url: The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1".
             model: The model identifier to use. Defaults to "gpt-oss-120b".
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=CerebrasLLMService.Settings(model=...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional keyword arguments passed to OpenAILLMService.
         """
-        super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(model="gpt-oss-120b")
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. (No step 3, as there's no params object to apply)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(api_key=api_key, base_url=base_url, settings=default_settings, **kwargs)
 
     def create_client(self, api_key=None, base_url=None, **kwargs):
         """Create OpenAI-compatible client for Cerebras API endpoint.
@@ -68,16 +101,28 @@ class CerebrasLLMService(OpenAILLMService):
             Dictionary of parameters for the chat completion request.
         """
         params = {
-            "model": self.model_name,
+            "model": self._settings.model,
             "stream": True,
-            "seed": self._settings["seed"],
-            "temperature": self._settings["temperature"],
-            "top_p": self._settings["top_p"],
-            "max_completion_tokens": self._settings["max_completion_tokens"],
+            "seed": self._settings.seed,
+            "temperature": self._settings.temperature,
+            "top_p": self._settings.top_p,
+            "max_completion_tokens": self._settings.max_completion_tokens,
         }
 
         # Messages, tools, tool_choice
         params.update(params_from_context)
 
-        params.update(self._settings["extra"])
+        params.update(self._settings.extra)
+
+        # Prepend system instruction if set
+        if self._settings.system_instruction:
+            messages = params.get("messages", [])
+            if messages and messages[0].get("role") == "system":
+                logger.warning(
+                    f"{self}: Both system_instruction and an initial system message in context are set. This may be unintended."
+                )
+            params["messages"] = [
+                {"role": "system", "content": self._settings.system_instruction}
+            ] + messages
+
         return params

src/pipecat/services/deepgram/__init__.py

@@ -9,6 +9,7 @@ import sys
 from pipecat.services import DeprecatedModuleProxy
 
 from .flux import *
+from .sagemaker import *
 from .stt import *
 from .tts import *
 

src/pipecat/services/deepgram/flux/stt.py

@@ -9,6 +9,7 @@
 import asyncio
 import json
 import time
+from dataclasses import dataclass, field
 from enum import Enum
 from typing import Any, AsyncGenerator, Dict, Optional
 from urllib.parse import urlencode
@@ -27,7 +28,7 @@ from pipecat.frames.frames import (
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
 )
-from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
 from pipecat.services.stt_service import WebsocketSTTService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
@@ -52,6 +53,8 @@ class FluxMessageType(str, Enum):
     RECEIVE_CONNECTED = "Connected"
     RECEIVE_FATAL_ERROR = "Error"
     TURN_INFO = "TurnInfo"
+    CONFIGURE_SUCCESS = "ConfigureSuccess"
+    CONFIGURE_FAILURE = "ConfigureFailure"
 
 
 class FluxEventType(str, Enum):
@@ -68,19 +71,59 @@ class FluxEventType(str, Enum):
     UPDATE = "Update"
 
 
+@dataclass
+class DeepgramFluxSTTSettings(STTSettings):
+    """Settings for DeepgramFluxSTTService.
+
+    Parameters:
+        eager_eot_threshold: EagerEndOfTurn/TurnResumed threshold. Off by default.
+            Lower values = more aggressive (faster response, more LLM calls).
+            Higher values = more conservative (slower response, fewer LLM calls).
+        eot_threshold: End-of-turn confidence required to finish a turn (default 0.7).
+        eot_timeout_ms: Time in ms after speech to finish a turn regardless of EOT
+            confidence (default 5000).
+        keyterm: Keyterms to boost recognition accuracy for specialized terminology.
+        min_confidence: Minimum confidence required to create a TranscriptionFrame.
+    """
+
+    eager_eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    eot_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    eot_timeout_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    keyterm: list | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    min_confidence: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
 class DeepgramFluxSTTService(WebsocketSTTService):
     """Deepgram Flux speech-to-text service.
 
     Provides real-time speech recognition using Deepgram's WebSocket API with Flux capabilities.
     Supports configurable models, VAD events, and various audio processing options
     including advanced turn detection and EagerEndOfTurn events for improved conversational AI performance.
+
+    Event handlers available (in addition to WebsocketSTTService events):
+
+    - on_speech_started(service): Deepgram detected start of speech
+    - on_utterance_end(service): Deepgram detected end of utterance
+    - on_end_of_turn(service): Deepgram detected end of turn (EOT)
+    - on_eager_end_of_turn(service): Deepgram predicted end of turn (EagerEOT)
+    - on_turn_resumed(service): User resumed speaking after EagerEOT
+
+    Example::
+
+        @stt.event_handler("on_end_of_turn")
+        async def on_end_of_turn(service):
+            ...
     """
 
+    Settings = DeepgramFluxSTTSettings
+    _settings: Settings
+    _CONFIGURE_FIELDS = {"keyterm", "eot_threshold", "eager_eot_threshold", "eot_timeout_ms"}
+
     class InputParams(BaseModel):
         """Configuration parameters for Deepgram Flux API.
 
-        This class defines all available connection parameters for the Deepgram Flux API
-        based on the official documentation.
+        .. deprecated:: 0.0.105
+            Use ``settings=DeepgramFluxSTTService.Settings(...)`` instead.
 
         Parameters:
             eager_eot_threshold: Optional. EagerEndOfTurn/TurnResumed are off by default.
@@ -113,10 +156,13 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         api_key: str,
         url: str = "wss://api.deepgram.com/v2/listen",
         sample_rate: Optional[int] = None,
-        model: str = "flux-general-en",
+        mip_opt_out: Optional[bool] = None,
+        model: Optional[str] = None,
         flux_encoding: str = "linear16",
+        tag: Optional[list] = None,
         params: Optional[InputParams] = None,
         should_interrupt: bool = True,
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Deepgram Flux STT service.
@@ -124,13 +170,25 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         Args:
             api_key: Deepgram API key for authentication. Required for API access.
             url: WebSocket URL for the Deepgram Flux API. Defaults to the preview endpoint.
-            sample_rate: Audio sample rate in Hz. If None, uses the rate from params or 16000.
-            model: Deepgram Flux model to use for transcription. Currently only supports "flux-general-en".
+            sample_rate: Audio sample rate in Hz. If None, uses the pipeline
+                sample rate.
+            mip_opt_out: Opt out of the Deepgram Model Improvement Program.
+            model: Deepgram Flux model to use for transcription.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepgramFluxSTTService.Settings(model=...)`` instead.
+
             flux_encoding: Audio encoding format required by Flux API. Must be "linear16".
                 Raw signed little-endian 16-bit PCM encoding.
+            tag: Tags to label requests for identification during usage reporting.
             params: InputParams instance containing detailed API configuration options.
-                If None, default parameters will be used.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepgramFluxSTTService.Settings(...)`` instead.
+
             should_interrupt: Determine whether the bot should be interrupted when Flux detects that the user is speaking.
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional arguments passed to the parent WebsocketSTTService class.
 
         Examples:
@@ -140,16 +198,15 @@ class DeepgramFluxSTTService(WebsocketSTTService):
 
             Advanced usage with custom parameters::
 
-                params = DeepgramFluxSTTService.InputParams(
-                    eager_eot_threshold=0.5,
-                    eot_threshold=0.8,
-                    keyterm=["AI", "machine learning", "neural network"],
-                    tag=["production", "voice-agent"]
-                )
                 stt = DeepgramFluxSTTService(
                     api_key="your-api-key",
-                    model="flux-general-en",
-                    params=params
+                    settings=DeepgramFluxSTTService.Settings(
+                        model="flux-general-en",
+                        eager_eot_threshold=0.5,
+                        eot_threshold=0.8,
+                        keyterm=["AI", "machine learning", "neural network"],
+                        tag=["production", "voice-agent"],
+                    ),
                 )
         """
         # Note: For DeepgramFluxSTTService, differently from other processes, we need to create
@@ -162,18 +219,56 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         # was never destroyed.
         # So we can keep it here as false, because inside the method send_with_retry, it will
         # already try to reconnect if needed.
-        super().__init__(sample_rate=sample_rate, reconnect_on_error=False, **kwargs)
 
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="flux-general-en",
+            language=Language.EN,
+            eager_eot_threshold=None,
+            eot_threshold=None,
+            eot_timeout_ms=None,
+            keyterm=[],
+            min_confidence=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.eager_eot_threshold = params.eager_eot_threshold
+                default_settings.eot_threshold = params.eot_threshold
+                default_settings.eot_timeout_ms = params.eot_timeout_ms
+                default_settings.keyterm = params.keyterm or []
+                if params.tag and tag is None:
+                    tag = params.tag
+                default_settings.min_confidence = params.min_confidence
+                if params.mip_opt_out is not None:
+                    mip_opt_out = params.mip_opt_out
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            reconnect_on_error=False,
+            settings=default_settings,
+            **kwargs,
+        )
         self._api_key = api_key
         self._url = url
-        self._model = model
-        self._params = params or DeepgramFluxSTTService.InputParams()
         self._should_interrupt = should_interrupt
-        self._flux_encoding = flux_encoding
-        # This is the currently only supported language
-        self._language = Language.EN
+        self._encoding = flux_encoding
+        self._mip_opt_out = mip_opt_out
+        self._tag = tag or []
         self._websocket_url = None
         self._receive_task = None
+
         # Flux event handlers
         self._register_event_handler("on_start_of_turn")
         self._register_event_handler("on_turn_resumed")
@@ -194,6 +289,8 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         Establishes the WebSocket connection to the Deepgram Flux API and starts
         the background task for receiving transcription results.
         """
+        await super()._connect()
+
         await self._connect_websocket()
 
     async def _disconnect(self):
@@ -202,6 +299,8 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         Gracefully disconnects from the Deepgram Flux API, cancels background tasks,
         and cleans up resources to prevent memory leaks.
         """
+        await super()._disconnect()
+
         try:
             await self._disconnect_websocket()
         except Exception as e:
@@ -314,6 +413,33 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         except Exception as e:
             await self.push_error(error_msg=f"Error sending closeStream: {e}", exception=e)
 
+    async def _send_configure(self, fields: set[str]):
+        """Send a Configure control message to update settings mid-stream.
+
+        Builds a Configure JSON message containing only the fields that changed
+        and sends it over the existing WebSocket connection.
+
+        Args:
+            fields: Set of changed field names to include in the message.
+        """
+        message: dict[str, Any] = {"type": "Configure"}
+
+        if "keyterm" in fields:
+            message["keyterms"] = self._settings.keyterm
+
+        thresholds: dict[str, Any] = {}
+        if "eot_threshold" in fields:
+            thresholds["eot_threshold"] = self._settings.eot_threshold
+        if "eager_eot_threshold" in fields:
+            thresholds["eager_eot_threshold"] = self._settings.eager_eot_threshold
+        if "eot_timeout_ms" in fields:
+            thresholds["eot_timeout_ms"] = self._settings.eot_timeout_ms
+        if thresholds:
+            message["thresholds"] = thresholds
+
+        logger.debug(f"{self}: sending Configure message: {message}")
+        await self._websocket.send(json.dumps(message))
+
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
 
@@ -322,6 +448,26 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         """
         return True
 
+    async def _update_settings(self, delta: Settings) -> dict[str, Any]:
+        """Apply a settings delta.
+
+        Configure-able fields (keyterm, eot_threshold, eager_eot_threshold,
+        eot_timeout_ms) are sent to Deepgram via a Configure WebSocket message.
+        Other fields are stored but cannot be applied to the active connection.
+        """
+        changed = await super()._update_settings(delta)
+
+        if not changed:
+            return changed
+
+        configure_fields = changed.keys() & self._CONFIGURE_FIELDS
+        if configure_fields and self._websocket and self._websocket.state is State.OPEN:
+            await self._send_configure(configure_fields)
+
+        self._warn_unhandled_updated_settings(changed.keys() - self._CONFIGURE_FIELDS)
+
+        return changed
+
     async def start(self, frame: StartFrame):
         """Start the Deepgram Flux STT service.
 
@@ -334,29 +480,29 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         await super().start(frame)
 
         url_params = [
-            f"model={self._model}",
+            f"model={self._settings.model}",
             f"sample_rate={self.sample_rate}",
-            f"encoding={self._flux_encoding}",
+            f"encoding={self._encoding}",
         ]
 
-        if self._params.eager_eot_threshold is not None:
-            url_params.append(f"eager_eot_threshold={self._params.eager_eot_threshold}")
+        if self._settings.eager_eot_threshold is not None:
+            url_params.append(f"eager_eot_threshold={self._settings.eager_eot_threshold}")
 
-        if self._params.eot_threshold is not None:
-            url_params.append(f"eot_threshold={self._params.eot_threshold}")
+        if self._settings.eot_threshold is not None:
+            url_params.append(f"eot_threshold={self._settings.eot_threshold}")
 
-        if self._params.eot_timeout_ms is not None:
-            url_params.append(f"eot_timeout_ms={self._params.eot_timeout_ms}")
+        if self._settings.eot_timeout_ms is not None:
+            url_params.append(f"eot_timeout_ms={self._settings.eot_timeout_ms}")
 
-        if self._params.mip_opt_out is not None:
-            url_params.append(f"mip_opt_out={str(self._params.mip_opt_out).lower()}")
+        if self._mip_opt_out is not None:
+            url_params.append(f"mip_opt_out={str(self._mip_opt_out).lower()}")
 
         # Add keyterm parameters (can have multiple)
-        for keyterm in self._params.keyterm:
+        for keyterm in self._settings.keyterm:
             url_params.append(urlencode({"keyterm": keyterm}))
 
         # Add tag parameters (can have multiple)
-        for tag_value in self._params.tag:
+        for tag_value in self._tag:
             url_params.append(urlencode({"tag": tag_value}))
 
         self._websocket_url = f"{self._url}?{'&'.join(url_params)}"
@@ -515,6 +661,14 @@ class DeepgramFluxSTTService(WebsocketSTTService):
                 await self._handle_fatal_error(data)
             case FluxMessageType.TURN_INFO:
                 await self._handle_turn_info(data)
+            case FluxMessageType.CONFIGURE_SUCCESS:
+                logger.info(f"{self}: Configure accepted: {data}")
+            case FluxMessageType.CONFIGURE_FAILURE:
+                error_code = data.get("error_code", "unknown")
+                description = data.get("description", "no description")
+                error_msg = f"Configure rejected: [{error_code}] {description}"
+                logger.warning(f"{self}: {error_msg}")
+                await self.push_error(error_msg=error_msg)
 
     async def _handle_connection_established(self):
         """Handle successful connection establishment to Deepgram Flux.
@@ -596,7 +750,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         self._user_is_speaking = True
         await self.broadcast_frame(UserStartedSpeakingFrame)
         if self._should_interrupt:
-            await self.push_interruption_task_frame_and_wait()
+            await self.broadcast_interruption()
         await self.start_metrics()
         await self._call_event_handler("on_start_of_turn", transcript)
         if transcript:
@@ -655,14 +809,17 @@ class DeepgramFluxSTTService(WebsocketSTTService):
         # Compute the average confidence
         average_confidence = self._calculate_average_confidence(data)
 
-        if not self._params.min_confidence or average_confidence > self._params.min_confidence:
+        if not self._settings.min_confidence or average_confidence > self._settings.min_confidence:
+            # EndOfTurn means Flux has determined the turn is complete,
+            # so this TranscriptionFrame is always finalized
             await self.push_frame(
                 TranscriptionFrame(
                     transcript,
                     self._user_id,
                     time_now_iso8601(),
-                    self._language,
+                    self._settings.language,
                     result=data,
+                    finalized=True,
                 )
             )
         else:
@@ -670,10 +827,9 @@ class DeepgramFluxSTTService(WebsocketSTTService):
                 f"Transcription confidence below min_confidence threshold: {average_confidence}"
             )
 
-        await self._handle_transcription(transcript, True, self._language)
+        await self._handle_transcription(transcript, True, self._settings.language)
         await self.stop_processing_metrics()
-        await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.DOWNSTREAM)
-        await self.push_frame(UserStoppedSpeakingFrame(), FrameDirection.UPSTREAM)
+        await self.broadcast_frame(UserStoppedSpeakingFrame)
         await self._call_event_handler("on_end_of_turn", transcript)
 
     async def _handle_eager_end_of_turn(self, transcript: str, data: Dict[str, Any]):
@@ -715,7 +871,7 @@ class DeepgramFluxSTTService(WebsocketSTTService):
                 transcript,
                 self._user_id,
                 time_now_iso8601(),
-                self._language,
+                self._settings.language,
                 result=data,
             )
         )

src/pipecat/services/deepgram/sagemaker/stt.py

@@ -0,0 +1,527 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Deepgram speech-to-text service for AWS SageMaker.
+
+This module provides a Pipecat STT service that connects to Deepgram models
+deployed on AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for
+low-latency real-time transcription with support for interim results, multiple
+languages, and various Deepgram features.
+"""
+
+import asyncio
+import json
+from dataclasses import dataclass, fields
+from typing import Any, AsyncGenerator, Optional
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    ErrorFrame,
+    Frame,
+    InterimTranscriptionFrame,
+    StartFrame,
+    TranscriptionFrame,
+    VADUserStartedSpeakingFrame,
+    VADUserStoppedSpeakingFrame,
+)
+from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
+from pipecat.services.deepgram.stt import DeepgramSTTService, LiveOptions
+from pipecat.services.settings import STTSettings, is_given
+from pipecat.services.stt_latency import DEEPGRAM_SAGEMAKER_TTFS_P99
+from pipecat.services.stt_service import STTService
+from pipecat.transcriptions.language import Language
+from pipecat.utils.time import time_now_iso8601
+from pipecat.utils.tracing.service_decorators import traced_stt
+
+
+@dataclass
+class DeepgramSageMakerSTTSettings(DeepgramSTTService.Settings):
+    """Settings for the Deepgram SageMaker STT service.
+
+    Inherits all fields from :class:`DeepgramSTTService.Settings`.
+    """
+
+    pass
+
+
+class DeepgramSageMakerSTTService(STTService):
+    """Deepgram speech-to-text service for AWS SageMaker.
+
+    Provides real-time speech recognition using Deepgram models deployed on
+    AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for low-latency
+    transcription with support for interim results, speaker diarization, and
+    multiple languages.
+
+    Requirements:
+
+    - AWS credentials configured (via environment variables, AWS CLI, or instance metadata)
+    - A deployed SageMaker endpoint with Deepgram model: https://developers.deepgram.com/docs/deploy-amazon-sagemaker
+
+    Example::
+
+        stt = DeepgramSageMakerSTTService(
+            endpoint_name="my-deepgram-endpoint",
+            region="us-east-2",
+            settings=DeepgramSageMakerSTTService.Settings(
+                model="nova-3",
+                language="en",
+                interim_results=True,
+                punctuate=True,
+            ),
+        )
+    """
+
+    Settings = DeepgramSageMakerSTTSettings
+    _settings: Settings
+
+    def __init__(
+        self,
+        *,
+        endpoint_name: str,
+        region: str,
+        encoding: str = "linear16",
+        channels: int = 1,
+        multichannel: bool = False,
+        sample_rate: Optional[int] = None,
+        mip_opt_out: Optional[bool] = None,
+        live_options: Optional[LiveOptions] = None,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = DEEPGRAM_SAGEMAKER_TTFS_P99,
+        **kwargs,
+    ):
+        """Initialize the Deepgram SageMaker STT service.
+
+        Args:
+            endpoint_name: Name of the SageMaker endpoint with Deepgram model
+                deployed (e.g., "my-deepgram-nova-3-endpoint").
+            region: AWS region where the endpoint is deployed (e.g., "us-east-2").
+            encoding: Audio encoding format. Defaults to "linear16".
+            channels: Number of audio channels. Defaults to 1.
+            multichannel: Transcribe each audio channel independently.
+                Defaults to False.
+            sample_rate: Audio sample rate in Hz. If None, uses the pipeline
+                sample rate.
+            mip_opt_out: Opt out of Deepgram model improvement program.
+            live_options: Legacy configuration options.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepgramSageMakerSTTService.Settings(...)`` for
+                    runtime-updatable fields and direct init parameters for
+                    connection-level config.
+
+            settings: Runtime-updatable settings. When provided alongside
+                ``live_options``, ``settings`` values take precedence (applied
+                after the ``live_options`` merge).
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
+            **kwargs: Additional arguments passed to the parent STTService.
+        """
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="nova-3",
+            language=Language.EN,
+            detect_entities=False,
+            diarize=False,
+            dictation=False,
+            endpointing=None,
+            interim_results=True,
+            keyterm=None,
+            keywords=None,
+            numerals=False,
+            profanity_filter=True,
+            punctuate=True,
+            redact=None,
+            replace=None,
+            search=None,
+            smart_format=False,
+            utterance_end_ms=None,
+            vad_events=False,
+        )
+
+        # 2. Apply live_options overrides — only if settings not provided
+        if live_options is not None:
+            self._warn_init_param_moved_to_settings("live_options")
+            if not settings:
+                # Extract init-only fields from live_options
+                if live_options.sample_rate is not None and sample_rate is None:
+                    sample_rate = live_options.sample_rate
+                if live_options.encoding is not None:
+                    encoding = live_options.encoding
+                if live_options.channels is not None:
+                    channels = live_options.channels
+                if live_options.multichannel is not None:
+                    multichannel = live_options.multichannel
+                if live_options.mip_opt_out is not None:
+                    mip_opt_out = live_options.mip_opt_out
+
+                # Build settings delta from remaining fields
+                init_only = {
+                    "sample_rate",
+                    "encoding",
+                    "channels",
+                    "multichannel",
+                    "mip_opt_out",
+                }
+                lo_dict = {k: v for k, v in live_options.to_dict().items() if k not in init_only}
+                delta = self.Settings.from_mapping(lo_dict)
+                default_settings.apply_update(delta)
+
+        # 3. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        # Sync extra to top-level fields so self._settings is unambiguous
+        default_settings._sync_extra_to_fields()
+
+        super().__init__(
+            sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            settings=default_settings,
+            **kwargs,
+        )
+
+        self._endpoint_name = endpoint_name
+        self._region = region
+
+        # Init-only connection config (not runtime-updatable).
+        self._encoding = encoding
+        self._channels = channels
+        self._multichannel = multichannel
+        self._mip_opt_out = mip_opt_out
+
+        self._client: Optional[SageMakerBidiClient] = None
+        self._response_task: Optional[asyncio.Task] = None
+        self._keepalive_task: Optional[asyncio.Task] = None
+
+    def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Deepgram SageMaker service supports metrics generation.
+        """
+        return True
+
+    async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
+        """Apply a settings delta and warn about unhandled changes."""
+        changed = await super()._update_settings(delta)
+
+        if not changed:
+            return changed
+
+        # Sync extra to fields after the update so self._settings stays unambiguous
+        if isinstance(self._settings, self.Settings):
+            self._settings._sync_extra_to_fields()
+
+        # TODO: someday we could reconnect here to apply updated settings.
+        # Code might look something like the below:
+        # await self._disconnect()
+        # await self._connect()
+
+        self._warn_unhandled_updated_settings(changed)
+
+        return changed
+
+    async def start(self, frame: StartFrame):
+        """Start the Deepgram SageMaker STT service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
+        await super().start(frame)
+        await self._connect()
+
+    async def stop(self, frame: EndFrame):
+        """Stop the Deepgram SageMaker STT service.
+
+        Args:
+            frame: The end frame.
+        """
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        """Cancel the Deepgram SageMaker STT service.
+
+        Args:
+            frame: The cancel frame.
+        """
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
+        """Send audio data to Deepgram for transcription.
+
+        Args:
+            audio: Raw audio bytes to transcribe.
+
+        Yields:
+            Frame: None (transcription results come via BiDi stream callbacks).
+        """
+        if self._client and self._client.is_active:
+            try:
+                await self._client.send_audio_chunk(audio)
+            except Exception as e:
+                yield ErrorFrame(error=f"Unknown error occurred: {e}")
+        yield None
+
+    def _build_query_string(self) -> str:
+        """Build query string from current settings and init-only connection config."""
+        params = {}
+        s = self._settings
+
+        # Declared Deepgram-specific fields from settings
+        for f in fields(s):
+            if f.name in ("model", "language", "extra") or f.name.startswith("_"):
+                continue
+            value = getattr(s, f.name)
+            if not is_given(value) or value is None:
+                continue
+            params[f.name] = str(value).lower() if isinstance(value, bool) else str(value)
+
+        # model and language
+        if is_given(s.model) and s.model is not None:
+            params["model"] = str(s.model)
+        if is_given(s.language) and s.language is not None:
+            params["language"] = str(s.language)
+
+        # Init-only connection config
+        params["encoding"] = self._encoding
+        params["channels"] = str(self._channels)
+        params["multichannel"] = str(self._multichannel).lower()
+        params["sample_rate"] = str(self.sample_rate)
+
+        if self._mip_opt_out is not None:
+            params["mip_opt_out"] = str(self._mip_opt_out).lower()
+
+        # Any remaining values in extra
+        if s.extra:
+            for key, value in s.extra.items():
+                if value is not None:
+                    params[key] = str(value).lower() if isinstance(value, bool) else str(value)
+
+        return "&".join(f"{k}={v}" for k, v in params.items())
+
+    async def _connect(self):
+        """Connect to the SageMaker endpoint and start the BiDi session.
+
+        Builds the Deepgram query string from settings, creates the BiDi client,
+        starts the streaming session, and launches background tasks for processing
+        responses and sending KeepAlive messages.
+        """
+        logger.debug("Connecting to Deepgram on SageMaker...")
+
+        query_string = self._build_query_string()
+
+        # Create BiDi client
+        self._client = SageMakerBidiClient(
+            endpoint_name=self._endpoint_name,
+            region=self._region,
+            model_invocation_path="v1/listen",
+            model_query_string=query_string,
+        )
+
+        try:
+            # Start the session
+            await self._client.start_session()
+
+            # Start processing responses in the background
+            self._response_task = self.create_task(self._process_responses())
+
+            # Start keepalive task to maintain connection
+            self._keepalive_task = self.create_task(self._send_keepalive())
+
+            logger.debug("Connected to Deepgram on SageMaker")
+            await self._call_event_handler("on_connected")
+
+        except Exception as e:
+            await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
+            await self._call_event_handler("on_connection_error", str(e))
+
+    async def _disconnect(self):
+        """Disconnect from the SageMaker endpoint.
+
+        Sends a CloseStream message to Deepgram, cancels background tasks
+        (KeepAlive and response processing), and closes the BiDi session.
+        Safe to call multiple times.
+        """
+        if self._client and self._client.is_active:
+            logger.debug("Disconnecting from Deepgram on SageMaker...")
+
+            # Send CloseStream message to Deepgram
+            try:
+                await self._client.send_json({"type": "CloseStream"})
+            except Exception as e:
+                logger.warning(f"Failed to send CloseStream message: {e}")
+
+            # Cancel keepalive task
+            if self._keepalive_task and not self._keepalive_task.done():
+                await self.cancel_task(self._keepalive_task)
+
+            # Cancel response processing task
+            if self._response_task and not self._response_task.done():
+                await self.cancel_task(self._response_task)
+
+            # Close the BiDi session
+            await self._client.close_session()
+
+            logger.debug("Disconnected from Deepgram on SageMaker")
+            await self._call_event_handler("on_disconnected")
+
+    async def _send_keepalive(self):
+        """Send periodic KeepAlive messages to maintain the connection.
+
+        Sends a KeepAlive JSON message to Deepgram every 5 seconds while the
+        connection is active. This prevents the connection from timing out during
+        periods of silence.
+        """
+        while self._client and self._client.is_active:
+            await asyncio.sleep(5)
+            if self._client and self._client.is_active:
+                try:
+                    await self._client.send_json({"type": "KeepAlive"})
+                except Exception as e:
+                    logger.warning(f"Failed to send KeepAlive: {e}")
+
+    async def _process_responses(self):
+        """Process streaming responses from Deepgram on SageMaker.
+
+        Continuously receives responses from the BiDi stream, decodes the payload,
+        parses JSON responses from Deepgram, and processes transcription results.
+        Runs as a background task until the connection is closed or cancelled.
+        """
+        try:
+            while self._client and self._client.is_active:
+                result = await self._client.receive_response()
+
+                if result is None:
+                    break
+
+                # Check if this is a PayloadPart with bytes
+                if hasattr(result, "value") and hasattr(result.value, "bytes_"):
+                    if result.value.bytes_:
+                        response_data = result.value.bytes_.decode("utf-8")
+
+                        try:
+                            # Parse JSON response from Deepgram
+                            parsed = json.loads(response_data)
+
+                            # Extract and process transcript if available
+                            if "channel" in parsed:
+                                await self._handle_transcript_response(parsed)
+
+                        except json.JSONDecodeError:
+                            logger.warning(f"Non-JSON response: {response_data}")
+
+        except asyncio.CancelledError:
+            logger.debug("Response processor cancelled")
+        except Exception as e:
+            await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
+        finally:
+            logger.debug("Response processor stopped")
+
+    async def _handle_transcript_response(self, parsed: dict):
+        """Handle a transcript response from Deepgram.
+
+        Extracts the transcript text, determines if it's final or interim, extracts
+        language information, and pushes the appropriate frame (TranscriptionFrame
+        or InterimTranscriptionFrame) downstream.
+
+        Args:
+            parsed: The parsed JSON response from Deepgram containing channel,
+                alternatives, transcript, and metadata.
+        """
+        alternatives = parsed.get("channel", {}).get("alternatives", [])
+        if not alternatives or not alternatives[0].get("transcript"):
+            return
+
+        transcript = alternatives[0]["transcript"]
+        if not transcript.strip():
+            return
+
+        is_final = parsed.get("is_final", False)
+
+        # Extract language if available
+        language = None
+        if alternatives[0].get("languages"):
+            language = alternatives[0]["languages"][0]
+            language = Language(language)
+
+        if is_final:
+            # Check if this response is from a finalize() call.
+            # Only mark as finalized when both we requested it AND Deepgram confirms it.
+            from_finalize = parsed.get("from_finalize", False)
+            if from_finalize:
+                self.confirm_finalize()
+            await self.push_frame(
+                TranscriptionFrame(
+                    transcript,
+                    self._user_id,
+                    time_now_iso8601(),
+                    language,
+                    result=parsed,
+                )
+            )
+            await self._handle_transcription(transcript, is_final, language)
+            await self.stop_processing_metrics()
+        else:
+            # Interim transcription
+            await self.push_frame(
+                InterimTranscriptionFrame(
+                    transcript,
+                    self._user_id,
+                    time_now_iso8601(),
+                    language,
+                    result=parsed,
+                )
+            )
+
+    @traced_stt
+    async def _handle_transcription(
+        self, transcript: str, is_final: bool, language: Optional[Language] = None
+    ):
+        """Handle a transcription result with tracing.
+
+        This method is decorated with @traced_stt for observability and tracing
+        integration. The actual transcription processing is handled by the parent
+        class and observers.
+
+        Args:
+            transcript: The transcribed text.
+            is_final: Whether this is a final transcription result.
+            language: The detected language of the transcription, if available.
+        """
+        pass
+
+    async def _start_metrics(self):
+        """Start processing metrics collection."""
+        await self.start_processing_metrics()
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process frames with Deepgram SageMaker-specific handling.
+
+        Args:
+            frame: The frame to process.
+            direction: The direction of frame processing.
+        """
+        await super().process_frame(frame, direction)
+
+        # Start metrics when user starts speaking (if VAD is not provided by Deepgram)
+        if isinstance(frame, VADUserStartedSpeakingFrame):
+            await self._start_metrics()
+        elif isinstance(frame, VADUserStoppedSpeakingFrame):
+            # https://developers.deepgram.com/docs/finalize
+            # Mark that we're awaiting a from_finalize response
+            self.request_finalize()
+            if self._client and self._client.is_active:
+                try:
+                    await self._client.send_json({"type": "Finalize"})
+                except Exception as e:
+                    logger.warning(f"Error sending Finalize message: {e}")
+            logger.trace(f"Triggered finalize event on: {frame.name=}, {direction=}")

src/pipecat/services/deepgram/sagemaker/tts.py

@@ -0,0 +1,342 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Deepgram text-to-speech service for AWS SageMaker.
+
+This module provides a Pipecat TTS service that connects to Deepgram models
+deployed on AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for
+low-latency real-time speech synthesis with support for interruptions and
+streaming audio output.
+"""
+
+import asyncio
+import json
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Optional
+
+from loguru import logger
+
+from pipecat.frames.frames import (
+    CancelFrame,
+    EndFrame,
+    ErrorFrame,
+    Frame,
+    StartFrame,
+    TTSAudioRawFrame,
+)
+from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
+from pipecat.services.settings import TTSSettings
+from pipecat.services.tts_service import TTSService
+from pipecat.utils.tracing.service_decorators import traced_tts
+
+
+@dataclass
+class DeepgramSageMakerTTSSettings(TTSSettings):
+    """Settings for DeepgramSageMakerTTSService."""
+
+    pass
+
+
+class DeepgramSageMakerTTSService(TTSService):
+    """Deepgram text-to-speech service for AWS SageMaker.
+
+    Provides real-time speech synthesis using Deepgram models deployed on
+    AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for low-latency
+    audio generation with support for interruptions via the Clear message.
+
+    Requirements:
+
+    - AWS credentials configured (via environment variables, AWS CLI, or instance metadata)
+    - A deployed SageMaker endpoint with Deepgram TTS model: https://developers.deepgram.com/docs/deploy-amazon-sagemaker
+    - ``pipecat-ai[sagemaker]`` installed
+
+    Example::
+
+        tts = DeepgramSageMakerTTSService(
+            endpoint_name="my-deepgram-tts-endpoint",
+            region="us-east-2",
+            settings=DeepgramSageMakerTTSService.Settings(
+                voice="aura-2-helena-en",
+            )
+        )
+    """
+
+    Settings = DeepgramSageMakerTTSSettings
+    _settings: Settings
+
+    def __init__(
+        self,
+        *,
+        endpoint_name: str,
+        region: str,
+        voice: Optional[str] = None,
+        sample_rate: Optional[int] = None,
+        encoding: str = "linear16",
+        settings: Optional[Settings] = None,
+        **kwargs,
+    ):
+        """Initialize the Deepgram SageMaker TTS service.
+
+        Args:
+            endpoint_name: Name of the SageMaker endpoint with Deepgram TTS model
+                deployed (e.g., "my-deepgram-tts-endpoint").
+            region: AWS region where the endpoint is deployed (e.g., "us-east-2").
+            voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en".
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepgramSageMakerTTSService.Settings(voice=...)`` instead.
+
+            sample_rate: Audio sample rate in Hz. If None, uses the value from StartFrame.
+            encoding: Audio encoding format. Defaults to "linear16".
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            **kwargs: Additional arguments passed to the parent TTSService.
+        """
+        if voice is not None:
+            self._warn_init_param_moved_to_settings("voice", "voice")
+
+        voice = voice or "aura-2-helena-en"
+
+        default_settings = self.Settings(
+            model=None,
+            voice=voice,
+            language=None,
+        )
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            pause_frame_processing=True,
+            append_trailing_space=True,
+            settings=default_settings,
+            **kwargs,
+        )
+
+        self._endpoint_name = endpoint_name
+        self._region = region
+        self._encoding = encoding
+
+        self._client: Optional[SageMakerBidiClient] = None
+        self._response_task: Optional[asyncio.Task] = None
+
+    def can_generate_metrics(self) -> bool:
+        """Check if this service can generate processing metrics.
+
+        Returns:
+            True, as Deepgram SageMaker TTS service supports metrics generation.
+        """
+        return True
+
+    async def start(self, frame: StartFrame):
+        """Start the Deepgram SageMaker TTS service.
+
+        Args:
+            frame: The start frame containing initialization parameters.
+        """
+        await super().start(frame)
+        await self._connect()
+
+    async def stop(self, frame: EndFrame):
+        """Stop the Deepgram SageMaker TTS service.
+
+        Args:
+            frame: The end frame.
+        """
+        await super().stop(frame)
+        await self._disconnect()
+
+    async def cancel(self, frame: CancelFrame):
+        """Cancel the Deepgram SageMaker TTS service.
+
+        Args:
+            frame: The cancel frame.
+        """
+        await super().cancel(frame)
+        await self._disconnect()
+
+    async def _connect(self):
+        """Connect to the SageMaker endpoint and start the BiDi session.
+
+        Builds the Deepgram TTS query string, creates the BiDi client,
+        starts the streaming session, and launches a background task for processing
+        responses.
+        """
+        logger.debug("Connecting to Deepgram TTS on SageMaker...")
+
+        query_string = (
+            f"model={self._settings.voice}&encoding={self._encoding}&sample_rate={self.sample_rate}"
+        )
+
+        self._client = SageMakerBidiClient(
+            endpoint_name=self._endpoint_name,
+            region=self._region,
+            model_invocation_path="v1/speak",
+            model_query_string=query_string,
+        )
+
+        try:
+            await self._client.start_session()
+
+            self._response_task = self.create_task(self._process_responses())
+
+            logger.debug("Connected to Deepgram TTS on SageMaker")
+            await self._call_event_handler("on_connected")
+
+        except Exception as e:
+            await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
+            await self._call_event_handler("on_connection_error", str(e))
+
+    async def _disconnect(self):
+        """Disconnect from the SageMaker endpoint.
+
+        Sends a Close message to Deepgram, cancels the response processing task,
+        and closes the BiDi session. Safe to call multiple times.
+        """
+        if self._client and self._client.is_active:
+            logger.debug("Disconnecting from Deepgram TTS on SageMaker...")
+
+            try:
+                await self._client.send_json({"type": "Close"})
+            except Exception as e:
+                logger.warning(f"Failed to send Close message: {e}")
+
+            if self._response_task and not self._response_task.done():
+                await self.cancel_task(self._response_task)
+
+            await self._client.close_session()
+
+            logger.debug("Disconnected from Deepgram TTS on SageMaker")
+            await self._call_event_handler("on_disconnected")
+
+    async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
+        """Apply a settings delta and reconnect if necessary.
+
+        Since all settings are part of the SageMaker session query string,
+        any setting change requires reconnecting to apply the new values.
+        """
+        changed = await super()._update_settings(delta)
+
+        if not changed:
+            return changed
+
+        # Deepgram uses voice as the model, so keep them in sync for metrics
+        if "voice" in changed:
+            self._settings.model = self._settings.voice
+            self._sync_model_name_to_metrics()
+
+        # TODO: someday we could reconnect here to apply updated settings.
+        # Code might look something like the below:
+        # await self._disconnect()
+        # await self._connect()
+
+        self._warn_unhandled_updated_settings(changed)
+
+        return changed
+
+    async def _process_responses(self):
+        """Process streaming responses from Deepgram TTS on SageMaker.
+
+        Continuously receives responses from the BiDi stream. Attempts to decode
+        each payload as UTF-8 JSON for control messages (Flushed, Cleared, Metadata,
+        Warning). If decoding fails, treats the payload as raw audio bytes and pushes
+        a TTSAudioRawFrame downstream.
+        """
+        try:
+            while self._client and self._client.is_active:
+                result = await self._client.receive_response()
+
+                if result is None:
+                    break
+
+                if hasattr(result, "value") and hasattr(result.value, "bytes_"):
+                    if result.value.bytes_:
+                        payload = result.value.bytes_
+
+                        # Try to decode as JSON control message first
+                        try:
+                            response_data = payload.decode("utf-8")
+                            parsed = json.loads(response_data)
+                            msg_type = parsed.get("type")
+
+                            if msg_type == "Metadata":
+                                logger.trace(f"Received metadata: {parsed}")
+                            elif msg_type == "Flushed":
+                                logger.trace(f"Received Flushed: {parsed}")
+                            elif msg_type == "Cleared":
+                                logger.trace(f"Received Cleared: {parsed}")
+                            elif msg_type == "Warning":
+                                logger.warning(
+                                    f"{self} warning: "
+                                    f"{parsed.get('description', 'Unknown warning')}"
+                                )
+                            else:
+                                logger.debug(f"Received unknown message type: {parsed}")
+
+                        except (UnicodeDecodeError, json.JSONDecodeError):
+                            # Not JSON — treat as raw audio bytes
+                            await self.stop_ttfb_metrics()
+                            context_id = self.get_active_audio_context_id()
+                            frame = TTSAudioRawFrame(
+                                payload,
+                                self.sample_rate,
+                                1,
+                                context_id=context_id,
+                            )
+                            await self.append_to_audio_context(context_id, frame)
+
+        except asyncio.CancelledError:
+            logger.debug("TTS response processor cancelled")
+        except Exception as e:
+            await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
+        finally:
+            logger.debug("TTS response processor stopped")
+
+    async def on_audio_context_interrupted(self, context_id: str):
+        """Called when an audio context is cancelled due to an interruption.
+
+        Args:
+            context_id: The ID of the audio context that was interrupted, or
+                ``None`` if no context was active at the time.
+        """
+        if self._client and self._client.is_active:
+            try:
+                await self._client.send_json({"type": "Clear"})
+            except Exception as e:
+                logger.error(f"{self} error sending Clear message: {e}")
+
+    async def flush_audio(self, context_id: Optional[str] = None):
+        """Flush any pending audio synthesis by sending Flush command.
+
+        This should be called when the LLM finishes a complete response to force
+        generation of audio from Deepgram's internal text buffer.
+        """
+        if self._client and self._client.is_active:
+            try:
+                await self._client.send_json({"type": "Flush"})
+            except Exception as e:
+                logger.error(f"{self} error sending Flush message: {e}")
+
+    @traced_tts
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
+        """Generate speech from text using Deepgram TTS on SageMaker.
+
+        Args:
+            text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
+
+        Yields:
+            Frame: TTSStartedFrame, then None (audio comes asynchronously via
+            the response processor).
+        """
+        logger.debug(f"{self}: Generating TTS [{text}]")
+        try:
+            await self._client.send_json({"type": "Speak", "text": text})
+            yield None
+        except Exception as e:
+            yield ErrorFrame(error=f"Unknown error occurred: {e}")

src/pipecat/services/deepgram/stt.py

@@ -6,7 +6,9 @@
 
 """Deepgram speech-to-text service implementation."""
 
-from typing import AsyncGenerator, Dict, Optional
+import asyncio
+from dataclasses import dataclass, field, fields
+from typing import Any, AsyncGenerator, Optional
 
 from loguru import logger
 
@@ -23,20 +25,25 @@ from pipecat.frames.frames import (
     VADUserStoppedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.settings import (
+    NOT_GIVEN,
+    STTSettings,
+    _NotGiven,
+    is_given,
+)
+from pipecat.services.stt_latency import DEEPGRAM_TTFS_P99
 from pipecat.services.stt_service import STTService
 from pipecat.transcriptions.language import Language
 from pipecat.utils.time import time_now_iso8601
 from pipecat.utils.tracing.service_decorators import traced_stt
 
 try:
-    from deepgram import (
-        AsyncListenWebSocketClient,
-        DeepgramClient,
-        DeepgramClientOptions,
-        ErrorResponse,
-        LiveOptions,
-        LiveResultResponse,
-        LiveTranscriptionEvents,
+    from deepgram import AsyncDeepgramClient
+    from deepgram.core.events import EventType
+    from deepgram.listen.v1.types import (
+        ListenV1Results,
+        ListenV1SpeechStarted,
+        ListenV1UtteranceEnd,
     )
 except ModuleNotFoundError as e:
     logger.error(f"Exception: {e}")
@@ -44,23 +51,281 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+class LiveOptions:
+    """Deepgram live transcription options.
+
+    Compatibility wrapper that mirrors the ``LiveOptions`` class removed in
+    deepgram-sdk v6.
+
+    .. deprecated:: 0.0.105
+        Use ``settings=DeepgramSTTService.Settings(...)`` for runtime-updatable fields
+        and direct ``__init__`` parameters for connection-level config instead.
+    """
+
+    def __init__(
+        self,
+        *,
+        callback: Optional[str] = None,
+        callback_method: Optional[str] = None,
+        channels: Optional[int] = None,
+        detect_entities: Optional[bool] = None,
+        diarize: Optional[bool] = None,
+        dictation: Optional[bool] = None,
+        encoding: Optional[str] = None,
+        endpointing: Optional[Any] = None,
+        extra: Optional[Any] = None,
+        interim_results: Optional[bool] = None,
+        keyterm: Optional[Any] = None,
+        keywords: Optional[Any] = None,
+        language: Optional[str] = None,
+        mip_opt_out: Optional[bool] = None,
+        model: Optional[str] = None,
+        multichannel: Optional[bool] = None,
+        numerals: Optional[bool] = None,
+        profanity_filter: Optional[bool] = None,
+        punctuate: Optional[bool] = None,
+        redact: Optional[Any] = None,
+        replace: Optional[Any] = None,
+        sample_rate: Optional[int] = None,
+        search: Optional[Any] = None,
+        smart_format: Optional[bool] = None,
+        tag: Optional[Any] = None,
+        utterance_end_ms: Optional[int] = None,
+        vad_events: Optional[bool] = None,
+        version: Optional[str] = None,
+        **kwargs,
+    ):
+        """Initialize live transcription options.
+
+        Args:
+            callback: Callback URL for async transcription delivery.
+            callback_method: HTTP method to use for the callback (``"GET"`` or ``"POST"``).
+            channels: Number of audio channels.
+            detect_entities: Enable named entity detection.
+            diarize: Enable speaker diarization.
+            dictation: Enable dictation mode (converts commands to punctuation).
+            encoding: Audio encoding (e.g. ``"linear16"``).
+            endpointing: Endpointing sensitivity in ms, or ``False`` to disable.
+            extra: Additional key-value metadata to attach to the transcription (str or list).
+            interim_results: Whether to emit interim transcriptions.
+            keyterm: Keyterms to boost (str or list of str).
+            keywords: Keywords to boost (str or list of str).
+            language: BCP-47 language tag (e.g. ``"en-US"``).
+            mip_opt_out: Opt out of model improvement program.
+            model: Deepgram model name (e.g. ``"nova-3-general"``).
+            multichannel: Enable per-channel transcription for multi-channel audio.
+            numerals: Convert spoken numbers to numerals.
+            profanity_filter: Filter profanity from transcripts.
+            punctuate: Add punctuation to transcripts.
+            redact: Redact sensitive information (str or list of redaction types).
+            replace: Word replacement rules (str or list).
+            sample_rate: Audio sample rate in Hz.
+            search: Search terms to highlight (str or list of str).
+            smart_format: Apply smart formatting to transcripts.
+            tag: Custom billing tag (str or list of str).
+            utterance_end_ms: Silence duration in ms before an utterance-end event.
+            vad_events: Enable Deepgram VAD speech-started / utterance-end events.
+            version: Model version (e.g. ``"latest"``).
+            **kwargs: Any additional Deepgram query parameters.
+        """
+        self.callback = callback
+        self.callback_method = callback_method
+        self.channels = channels
+        self.detect_entities = detect_entities
+        self.diarize = diarize
+        self.dictation = dictation
+        self.encoding = encoding
+        self.endpointing = endpointing
+        self.extra = extra
+        self.interim_results = interim_results
+        self.keyterm = keyterm
+        self.keywords = keywords
+        self.language = language
+        self.mip_opt_out = mip_opt_out
+        self.model = model
+        self.multichannel = multichannel
+        self.numerals = numerals
+        self.profanity_filter = profanity_filter
+        self.punctuate = punctuate
+        self.redact = redact
+        self.replace = replace
+        self.sample_rate = sample_rate
+        self.search = search
+        self.smart_format = smart_format
+        self.tag = tag
+        self.utterance_end_ms = utterance_end_ms
+        self.vad_events = vad_events
+        self.version = version
+        self._extra = kwargs
+
+    def __getattr__(self, name: str):
+        # Fall back to _extra for any params passed as **kwargs.
+        # __getattr__ is only called when normal attribute lookup fails.
+        extra = self.__dict__.get("_extra", {})
+        try:
+            return extra[name]
+        except KeyError:
+            raise AttributeError(f"'LiveOptions' object has no attribute '{name}'")
+
+    def to_dict(self) -> dict:
+        """Return a dict of all non-None options."""
+        result = {k: v for k, v in vars(self).items() if not k.startswith("_") and v is not None}
+        result.update({k: v for k, v in self._extra.items() if v is not None})
+        return result
+
+
+@dataclass
+class DeepgramSTTSettings(STTSettings):
+    """Settings for DeepgramSTTService.
+
+    ``model`` and ``language`` are inherited from ``STTSettings`` /
+    ``ServiceSettings``.  Additional Deepgram connection params may
+    be passed in through ``extra`` (also inherited).
+
+    Parameters:
+        detect_entities: Enable named entity detection.
+        diarize: Enable speaker diarization.
+        dictation: Enable dictation mode (converts commands to punctuation).
+        endpointing: Endpointing sensitivity in ms, or ``False`` to disable.
+        interim_results: Whether to emit interim transcriptions.
+        keyterm: Keyterms to boost (str or list of str).
+        keywords: Keywords to boost (str or list of str).
+        numerals: Convert spoken numbers to numerals.
+        profanity_filter: Filter profanity from transcripts.
+        punctuate: Add punctuation to transcripts.
+        redact: Redact sensitive information (str or list of redaction types).
+        replace: Word replacement rules (str or list).
+        search: Search terms to highlight (str or list of str).
+        smart_format: Apply smart formatting to transcripts.
+        utterance_end_ms: Silence duration in ms before an utterance-end event.
+        vad_events: Enable Deepgram VAD speech-started / utterance-end events.
+    """
+
+    detect_entities: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    diarize: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    dictation: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    endpointing: Any | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    interim_results: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    keyterm: Any | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    keywords: Any | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    numerals: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    profanity_filter: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    punctuate: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    redact: Any | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    replace: Any | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    search: Any | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    smart_format: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    utterance_end_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    vad_events: bool | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+    def _sync_extra_to_fields(self) -> None:
+        """Sync values from extra dict to declared fields.
+
+        If a key in extra matches a field name and the field is NOT_GIVEN,
+        promote the extra value to the field. This ensures self._settings
+        always reflects the "final truth" of values that will be used.
+
+        Keys in extra that match declared fields are always removed from extra
+        to avoid confusion, even if the field was already set.
+        """
+        if not self.extra:
+            return
+
+        field_names = {
+            f.name
+            for f in fields(self)
+            if f.name not in ("extra", "model", "language") and not f.name.startswith("_")
+        }
+
+        for key in list(self.extra.keys()):
+            if key in field_names:
+                current_value = getattr(self, key)
+                if not is_given(current_value):
+                    # Promote extra value to the field
+                    setattr(self, key, self.extra[key])
+                # Always remove from extra to avoid ambiguity
+                del self.extra[key]
+
+
+def _derive_deepgram_urls(base_url: str) -> tuple[str, str]:
+    """Derive paired WebSocket and HTTP URLs from a single base URL.
+
+    The Deepgram SDK client requires both a WebSocket URL (for streaming)
+    and an HTTP URL (for REST calls). This helper lets developers provide
+    a single ``base_url`` and consistently derives both, preserving the
+    security level they chose. Useful for air-gapped or private deployments
+    where insecure schemes (ws:// / http://) are acceptable.
+
+    Accepted inputs:
+        - ``wss://`` or ``https://`` — secure (paired as wss + https)
+        - ``ws://`` or ``http://`` — insecure (paired as ws + http)
+        - Bare hostname (no scheme) — defaults to secure
+        - Unrecognized scheme — logs a warning, defaults to secure
+
+    Args:
+        base_url: Host with optional scheme, port, and path.
+
+    Returns:
+        A (ws_url, http_url) tuple with consistent schemes.
+    """
+    known_schemes = ("wss://", "https://", "ws://", "http://")
+    if "://" in base_url:
+        scheme, host = base_url.split("://", 1)
+        scheme += "://"
+        if scheme not in known_schemes:
+            logger.warning(
+                f"Unrecognized scheme in base_url '{base_url}', defaulting to wss:// / https://"
+            )
+    else:
+        scheme = ""
+        host = base_url
+
+    insecure = scheme in ("ws://", "http://")
+    ws_url = f"{'ws' if insecure else 'wss'}://{host}"
+    http_url = f"{'http' if insecure else 'https'}://{host}"
+    return ws_url, http_url
+
+
 class DeepgramSTTService(STTService):
     """Deepgram speech-to-text service.
 
     Provides real-time speech recognition using Deepgram's WebSocket API.
     Supports configurable models, languages, and various audio processing options.
+
+    Event handlers available (in addition to STTService events):
+
+    - on_speech_started(service): Deepgram detected start of speech
+    - on_utterance_end(service): Deepgram detected end of utterance
+
+    Example::
+
+        @stt.event_handler("on_speech_started")
+        async def on_speech_started(service):
+            ...
     """
 
+    Settings = DeepgramSTTSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
         api_key: str,
         url: str = "",
         base_url: str = "",
+        encoding: str = "linear16",
+        channels: int = 1,
+        multichannel: bool = False,
         sample_rate: Optional[int] = None,
+        callback: Optional[str] = None,
+        callback_method: Optional[str] = None,
+        tag: Optional[Any] = None,
+        mip_opt_out: Optional[bool] = None,
         live_options: Optional[LiveOptions] = None,
-        addons: Optional[Dict] = None,
+        addons: Optional[dict] = None,
         should_interrupt: bool = True,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = DEEPGRAM_TTFS_P99,
         **kwargs,
     ):
         """Initialize the Deepgram STT service.
@@ -73,22 +338,39 @@ class DeepgramSTTService(STTService):
                     Parameter `url` is deprecated, use `base_url` instead.
 
             base_url: Custom Deepgram API base URL.
-            sample_rate: Audio sample rate. If None, uses default or live_options value.
-            live_options: Deepgram LiveOptions for detailed configuration.
+            encoding: Audio encoding format. Defaults to "linear16".
+            channels: Number of audio channels. Defaults to 1.
+            multichannel: Transcribe each audio channel independently.
+                Defaults to False.
+            sample_rate: Audio sample rate in Hz. If None, uses the pipeline
+                sample rate.
+            callback: Callback URL for async transcription delivery.
+            callback_method: HTTP method for the callback (``"GET"`` or ``"POST"``).
+            tag: Custom billing tag.
+            mip_opt_out: Opt out of Deepgram model improvement program.
+            live_options: Legacy configuration options.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepgramSTTService.Settings(...)`` for runtime-updatable
+                    fields and direct init parameters for connection-level config.
+
             addons: Additional Deepgram features to enable.
-            should_interrupt: Determine whether the bot should be interrupted when Deepgram VAD events are enabled and the system detects that the user is speaking.
+            should_interrupt: Whether to interrupt the bot when Deepgram VAD
+                detects the user is speaking.
 
                 .. deprecated:: 0.0.99
                     This parameter will be removed along with `vad_events` support.
 
+            settings: Runtime-updatable settings. When provided alongside
+                ``live_options``, ``settings`` values take precedence (applied
+                after the ``live_options`` merge).
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
             **kwargs: Additional arguments passed to the parent STTService.
 
         Note:
             The `vad_events` option in LiveOptions is deprecated as of version 0.0.99 and will be removed in a future version. Please use the Silero VAD instead.
         """
-        sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
-        super().__init__(sample_rate=sample_rate, **kwargs)
-
         if url:
             import warnings
 
@@ -100,36 +382,92 @@ class DeepgramSTTService(STTService):
                 )
             base_url = url
 
-        default_options = LiveOptions(
-            encoding="linear16",
-            language=Language.EN,
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
             model="nova-3-general",
-            channels=1,
+            language=Language.EN,
+            detect_entities=False,
+            diarize=False,
+            dictation=False,
+            endpointing=None,
             interim_results=True,
-            smart_format=True,
-            punctuate=True,
+            keyterm=None,
+            keywords=None,
+            numerals=False,
             profanity_filter=True,
+            punctuate=True,
+            redact=None,
+            replace=None,
+            search=None,
+            smart_format=False,
+            utterance_end_ms=None,
             vad_events=False,
         )
 
-        merged_options = default_options.to_dict()
-        if live_options:
-            default_model = default_options.model
-            merged_options.update(live_options.to_dict())
-            # NOTE(aleix): Fixes an in deepgram-sdk where `model` is initialized
-            # to the string "None" instead of the value `None`.
-            if "model" in merged_options and merged_options["model"] == "None":
-                merged_options["model"] = default_model
+        # 2. (No step 2, as there are no deprecated direct args)
 
-        if "language" in merged_options and isinstance(merged_options["language"], Language):
-            merged_options["language"] = merged_options["language"].value
+        # 3. Apply live_options overrides — only if settings not provided
+        if live_options is not None:
+            self._warn_init_param_moved_to_settings("live_options")
+            if not settings:
+                # Extract init-only fields from live_options
+                if live_options.sample_rate is not None and sample_rate is None:
+                    sample_rate = live_options.sample_rate
+                if live_options.encoding is not None:
+                    encoding = live_options.encoding
+                if live_options.channels is not None:
+                    channels = live_options.channels
+                if live_options.callback is not None:
+                    callback = live_options.callback
+                if live_options.callback_method is not None:
+                    callback_method = live_options.callback_method
+                if live_options.tag is not None:
+                    tag = live_options.tag
+                if live_options.mip_opt_out is not None:
+                    mip_opt_out = live_options.mip_opt_out
+                if live_options.multichannel is not None:
+                    multichannel = live_options.multichannel
+
+                # Build settings delta from remaining fields
+                init_only = {
+                    "sample_rate",
+                    "encoding",
+                    "channels",
+                    "multichannel",
+                    "callback",
+                    "callback_method",
+                    "tag",
+                    "mip_opt_out",
+                }
+                lo_dict = {k: v for k, v in live_options.to_dict().items() if k not in init_only}
+                delta = self.Settings.from_mapping(lo_dict)
+                default_settings.apply_update(delta)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        # Sync extra to top-level fields so self._settings is unambiguous
+        default_settings._sync_extra_to_fields()
+
+        super().__init__(
+            sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            settings=default_settings,
+            **kwargs,
+        )
 
-        self.set_model_name(merged_options["model"])
-        self._settings = merged_options
         self._addons = addons
         self._should_interrupt = should_interrupt
+        self._encoding = encoding
+        self._channels = channels
+        self._multichannel = multichannel
+        self._callback = callback
+        self._callback_method = callback_method
+        self._tag = tag
+        self._mip_opt_out = mip_opt_out
 
-        if merged_options.get("vad_events"):
+        if self._settings.vad_events:
             import warnings
 
             with warnings.catch_warnings():
@@ -141,13 +479,28 @@ class DeepgramSTTService(STTService):
                     stacklevel=2,
                 )
 
-        self._client = DeepgramClient(
-            api_key,
-            config=DeepgramClientOptions(
-                url=base_url,
-                options={"keepalive": "true"},  # verbose=logging.DEBUG
-            ),
-        )
+        # Build client - support optional custom base URL via DeepgramClientEnvironment
+        if base_url:
+            try:
+                from deepgram import DeepgramClientEnvironment
+
+                ws_url, http_url = _derive_deepgram_urls(base_url)
+                environment = DeepgramClientEnvironment(
+                    base=http_url,
+                    production=ws_url,
+                    agent=ws_url,
+                )
+                self._client = AsyncDeepgramClient(api_key=api_key, environment=environment)
+            except Exception:
+                logger.warning(
+                    f"{self}: Custom base_url configuration failed, falling back to default"
+                )
+                self._client = AsyncDeepgramClient(api_key=api_key)
+        else:
+            self._client = AsyncDeepgramClient(api_key=api_key)
+
+        self._connection = None
+        self._connection_task = None
 
         if self.vad_enabled:
             self._register_event_handler("on_speech_started")
@@ -160,7 +513,7 @@ class DeepgramSTTService(STTService):
         Returns:
             True if VAD events are enabled in the current settings.
         """
-        return self._settings["vad_events"]
+        return self._settings.vad_events
 
     def can_generate_metrics(self) -> bool:
         """Check if this service can generate processing metrics.
@@ -170,28 +523,22 @@ class DeepgramSTTService(STTService):
         """
         return True
 
-    async def set_model(self, model: str):
-        """Set the Deepgram model and reconnect.
+    async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
+        """Apply a settings delta and reconnect if anything changed."""
+        changed = await super()._update_settings(delta)
 
-        Args:
-            model: The Deepgram model name to use.
-        """
-        await super().set_model(model)
-        logger.info(f"Switching STT model to: [{model}]")
-        self._settings["model"] = model
-        await self._disconnect()
-        await self._connect()
+        if not changed:
+            return changed
 
-    async def set_language(self, language: Language):
-        """Set the recognition language and reconnect.
+        # Sync extra to fields after the update so self._settings stays unambiguous
+        if isinstance(self._settings, self.Settings):
+            self._settings._sync_extra_to_fields()
 
-        Args:
-            language: The language to use for speech recognition.
-        """
-        logger.info(f"Switching STT language to: [{language}]")
-        self._settings["language"] = language
-        await self._disconnect()
-        await self._connect()
+        if self._connection:
+            await self._disconnect()
+            await self._connect()
+
+        return changed
 
     async def start(self, frame: StartFrame):
         """Start the Deepgram STT service.
@@ -200,7 +547,6 @@ class DeepgramSTTService(STTService):
             frame: The start frame containing initialization parameters.
         """
         await super().start(frame)
-        self._settings["sample_rate"] = self.sample_rate
         await self._connect()
 
     async def stop(self, frame: EndFrame):
@@ -230,76 +576,154 @@ class DeepgramSTTService(STTService):
         Yields:
             Frame: None (transcription results come via WebSocket callbacks).
         """
-        await self._connection.send(audio)
+        if self._connection:
+            await self._connection.send_media(audio)
         yield None
 
+    def _build_connect_kwargs(self) -> dict:
+        """Build keyword arguments for ``client.listen.v1.connect()`` from current settings."""
+        kwargs = {}
+        s = self._settings
+
+        # Declared Deepgram-specific fields
+        for f in fields(s):
+            if f.name in ("model", "language", "extra") or f.name.startswith("_"):
+                continue
+            value = getattr(s, f.name)
+            if not is_given(value) or value is None:
+                continue
+            # Lists (e.g. keyterm, keywords, search, redact, replace) must be
+            # passed through as-is so the SDK's encode_query produces repeated
+            # query params (keyterm=a&keyterm=b) instead of a stringified list.
+            if isinstance(value, list):
+                kwargs[f.name] = value
+            elif isinstance(value, bool):
+                kwargs[f.name] = str(value).lower()
+            else:
+                kwargs[f.name] = str(value)
+
+        # model and language
+        if is_given(s.model) and s.model is not None:
+            kwargs["model"] = str(s.model)
+        if is_given(s.language) and s.language is not None:
+            kwargs["language"] = str(s.language)
+
+        # Init-only connection config
+        kwargs["encoding"] = self._encoding
+        kwargs["channels"] = str(self._channels)
+        kwargs["multichannel"] = str(self._multichannel).lower()
+        kwargs["sample_rate"] = str(self.sample_rate)
+
+        if self._callback is not None:
+            kwargs["callback"] = self._callback
+        if self._callback_method is not None:
+            kwargs["callback_method"] = self._callback_method
+        if self._tag is not None:
+            kwargs["tag"] = str(self._tag)
+        if self._mip_opt_out is not None:
+            kwargs["mip_opt_out"] = str(self._mip_opt_out).lower()
+
+        # Any remaining values in extra (that didn't map to declared fields)
+        for key, value in s.extra.items():
+            if value is not None:
+                if isinstance(value, list):
+                    kwargs[key] = value
+                elif isinstance(value, bool):
+                    kwargs[key] = str(value).lower()
+                else:
+                    kwargs[key] = str(value)
+
+        if self._addons:
+            for key, value in self._addons.items():
+                kwargs[key] = str(value)
+
+        return kwargs
+
     async def _connect(self):
         logger.debug("Connecting to Deepgram")
-
-        self._connection: AsyncListenWebSocketClient = self._client.listen.asyncwebsocket.v("1")
-
-        self._connection.on(
-            LiveTranscriptionEvents(LiveTranscriptionEvents.Transcript), self._on_message
-        )
-        self._connection.on(LiveTranscriptionEvents(LiveTranscriptionEvents.Error), self._on_error)
-
-        if self.vad_enabled:
-            self._connection.on(
-                LiveTranscriptionEvents(LiveTranscriptionEvents.SpeechStarted),
-                self._on_speech_started,
-            )
-            self._connection.on(
-                LiveTranscriptionEvents(LiveTranscriptionEvents.UtteranceEnd),
-                self._on_utterance_end,
-            )
-
-        if not await self._connection.start(options=self._settings, addons=self._addons):
-            await self.push_error(error_msg=f"Unable to connect to Deepgram")
-        else:
-            headers = {
-                k: v
-                for k, v in self._connection._socket.response.headers.items()
-                if k.startswith("dg-")
-            }
-            logger.debug(f'{self}: Websocket connection initialized: {{"headers": {headers}}}')
+        self._connection_task = self.create_task(self._connection_handler())
 
     async def _disconnect(self):
-        if await self._connection.is_connected():
-            logger.debug("Disconnecting from Deepgram")
-            # Deepgram swallows asyncio.CancelledError internally which prevents
-            # proper cancellation propagation. This issue was found with
-            # parallel pipelines where `CancelFrame` was not awaited for to
-            # finish in all branches and it was pushed downstream reaching the
-            # end of the pipeline, which caused `cleanup()` to be called while
-            # Deepgram disconnection was still finishing and therefore
-            # preventing the task cancellation that occurs during `cleanup()`.
-            # GH issue: https://github.com/deepgram/deepgram-python-sdk/issues/570
-            await self._connection.finish()
+        if not self._connection_task:
+            return
 
-    async def start_metrics(self):
-        """Start TTFB and processing metrics collection."""
-        await self.start_ttfb_metrics()
+        logger.debug("Disconnecting from Deepgram")
+        # Clear self._connection first to prevent run_stt from sending audio
+        # during the close handshake, then close gracefully on the saved ref.
+        connection = self._connection
+        self._connection = None
+
+        if connection:
+            await connection.send_close_stream()
+
+        await self.cancel_task(self._connection_task)
+        self._connection_task = None
+
+    async def _connection_handler(self):
+        """Manages the full WebSocket lifecycle inside a single async with block.
+
+        Reconnects automatically after transient errors. Exits cleanly when
+        the task is cancelled (i.e. on stop/cancel).
+        """
+        while True:
+            connect_kwargs = self._build_connect_kwargs()
+            try:
+                async with self._client.listen.v1.connect(**connect_kwargs) as connection:
+                    self._connection = connection
+                    connection.on(EventType.MESSAGE, self._on_message)
+                    connection.on(EventType.ERROR, self._on_error)
+
+                    logger.debug(f"{self}: Websocket connection initialized")
+
+                    keepalive_task = self.create_task(
+                        self._keepalive_handler(), f"{self}::keepalive"
+                    )
+                    try:
+                        await connection.start_listening()
+                    finally:
+                        await self.cancel_task(keepalive_task)
+            except asyncio.CancelledError:
+                raise
+            except Exception as e:
+                logger.warning(f"{self}: Connection lost, will retry: {e}")
+            finally:
+                self._connection = None
+
+    async def _keepalive_handler(self):
+        """Periodically send KeepAlive frames to prevent server-side timeout.
+
+        Deepgram closes inactive connections after 10 seconds (NET-0001 error).
+        Sending every 5 seconds stays within the recommended 3-5 second interval.
+        """
+        while True:
+            await asyncio.sleep(5)
+            if self._connection:
+                try:
+                    await self._connection.send_keep_alive()
+                    logger.trace(f"{self}: Sent keepalive")
+                except Exception as e:
+                    logger.warning(f"{self}: Keepalive failed: {e}")
+
+    async def _start_metrics(self):
+        """Start processing metrics collection for this utterance."""
         await self.start_processing_metrics()
 
-    async def _on_error(self, *args, **kwargs):
-        error: ErrorResponse = kwargs["error"]
+    async def _on_error(self, error):
         logger.warning(f"{self} connection error, will retry: {error}")
         await self.push_error(error_msg=f"{error}")
         await self.stop_all_metrics()
-        # NOTE(aleix): we don't disconnect (i.e. call finish on the connection)
-        # because this triggers more errors internally in the Deepgram SDK. So,
-        # we just forget about the previous connection and create a new one.
-        await self._connect()
+        # Reconnection is handled automatically by the retry loop in
+        # _connection_handler once start_listening() exits after the error.
 
-    async def _on_speech_started(self, *args, **kwargs):
-        await self.start_metrics()
-        await self._call_event_handler("on_speech_started", *args, **kwargs)
+    async def _on_speech_started(self, message):
+        await self._start_metrics()
+        await self._call_event_handler("on_speech_started", message)
         await self.broadcast_frame(UserStartedSpeakingFrame)
         if self._should_interrupt:
-            await self.push_interruption_task_frame_and_wait()
+            await self.broadcast_interruption()
 
-    async def _on_utterance_end(self, *args, **kwargs):
-        await self._call_event_handler("on_utterance_end", *args, **kwargs)
+    async def _on_utterance_end(self, message):
+        await self._call_event_handler("on_utterance_end", message)
         await self.broadcast_frame(UserStoppedSpeakingFrame)
 
     @traced_stt
@@ -309,41 +733,51 @@ class DeepgramSTTService(STTService):
         """Handle a transcription result with tracing."""
         pass
 
-    async def _on_message(self, *args, **kwargs):
-        result: LiveResultResponse = kwargs["result"]
-        if len(result.channel.alternatives) == 0:
-            return
-        is_final = result.is_final
-        transcript = result.channel.alternatives[0].transcript
-        language = None
-        if result.channel.alternatives[0].languages:
-            language = result.channel.alternatives[0].languages[0]
-            language = Language(language)
-        if len(transcript) > 0:
-            await self.stop_ttfb_metrics()
-            if is_final:
-                await self.push_frame(
-                    TranscriptionFrame(
-                        transcript,
-                        self._user_id,
-                        time_now_iso8601(),
-                        language,
-                        result=result,
+    async def _on_message(self, message):
+        if isinstance(message, ListenV1SpeechStarted):
+            if self.vad_enabled:
+                await self._on_speech_started(message)
+        elif isinstance(message, ListenV1UtteranceEnd):
+            if self.vad_enabled:
+                await self._on_utterance_end(message)
+        elif isinstance(message, ListenV1Results):
+            if not message.channel or len(message.channel.alternatives) == 0:
+                return
+            is_final = message.is_final
+            transcript = message.channel.alternatives[0].transcript
+            language = None
+            if message.channel.alternatives[0].languages:
+                language = message.channel.alternatives[0].languages[0]
+                language = Language(language)
+            if len(transcript) > 0:
+                if is_final:
+                    # Check if this response is from a finalize() call.
+                    # Only mark as finalized when both we requested it AND Deepgram confirms it.
+                    from_finalize = getattr(message, "from_finalize", False) or False
+                    if from_finalize:
+                        self.confirm_finalize()
+                    await self.push_frame(
+                        TranscriptionFrame(
+                            transcript,
+                            self._user_id,
+                            time_now_iso8601(),
+                            language,
+                            result=message,
+                        )
                     )
-                )
-                await self._handle_transcription(transcript, is_final, language)
-                await self.stop_processing_metrics()
-            else:
-                # For interim transcriptions, just push the frame without tracing
-                await self.push_frame(
-                    InterimTranscriptionFrame(
-                        transcript,
-                        self._user_id,
-                        time_now_iso8601(),
-                        language,
-                        result=result,
+                    await self._handle_transcription(transcript, is_final, language)
+                    await self.stop_processing_metrics()
+                else:
+                    # For interim transcriptions, just push the frame without tracing
+                    await self.push_frame(
+                        InterimTranscriptionFrame(
+                            transcript,
+                            self._user_id,
+                            time_now_iso8601(),
+                            language,
+                            result=message,
+                        )
                     )
-                )
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         """Process frames with Deepgram-specific handling.
@@ -356,8 +790,11 @@ class DeepgramSTTService(STTService):
 
         if isinstance(frame, VADUserStartedSpeakingFrame) and not self.vad_enabled:
             # Start metrics if Deepgram VAD is disabled & pipeline VAD has detected speech
-            await self.start_metrics()
+            await self._start_metrics()
         elif isinstance(frame, VADUserStoppedSpeakingFrame):
             # https://developers.deepgram.com/docs/finalize
-            await self._connection.finalize()
-            logger.trace(f"Triggered finalize event on: {frame.name=}, {direction=}")
+            # Mark that we're awaiting a from_finalize response
+            if self._connection:
+                self.request_finalize()
+                await self._connection.send_finalize()
+                logger.trace(f"Triggered finalize event on: {frame.name=}, {direction=}")

src/pipecat/services/deepgram/stt_sagemaker.py

@@ -4,441 +4,15 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-"""Deepgram speech-to-text service for AWS SageMaker.
+"""Deprecated: use ``pipecat.services.deepgram.sagemaker.stt`` instead."""
 
-This module provides a Pipecat STT service that connects to Deepgram models
-deployed on AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for
-low-latency real-time transcription with support for interim results, multiple
-languages, and various Deepgram features.
-"""
+import warnings
 
-import asyncio
-import json
-from typing import AsyncGenerator, Optional
-
-from loguru import logger
-
-from pipecat.frames.frames import (
-    CancelFrame,
-    EndFrame,
-    ErrorFrame,
-    Frame,
-    InterimTranscriptionFrame,
-    StartFrame,
-    TranscriptionFrame,
-    VADUserStartedSpeakingFrame,
-    VADUserStoppedSpeakingFrame,
+warnings.warn(
+    "Module `pipecat.services.deepgram.stt_sagemaker` is deprecated, "
+    "use `pipecat.services.deepgram.sagemaker.stt` instead.",
+    DeprecationWarning,
+    stacklevel=2,
 )
-from pipecat.processors.frame_processor import FrameDirection
-from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
-from pipecat.services.stt_service import STTService
-from pipecat.transcriptions.language import Language
-from pipecat.utils.time import time_now_iso8601
-from pipecat.utils.tracing.service_decorators import traced_stt
 
-try:
-    from deepgram import LiveOptions
-except ModuleNotFoundError as e:
-    logger.error(f"Exception: {e}")
-    logger.error(
-        "In order to use DeepgramSageMakerSTTService, you need to `pip install pipecat-ai[deepgram,sagemaker]`."
-    )
-    raise Exception(f"Missing module: {e}")
-
-
-class DeepgramSageMakerSTTService(STTService):
-    """Deepgram speech-to-text service for AWS SageMaker.
-
-    Provides real-time speech recognition using Deepgram models deployed on
-    AWS SageMaker endpoints. Uses HTTP/2 bidirectional streaming for low-latency
-    transcription with support for interim results, speaker diarization, and
-    multiple languages.
-
-    Requirements:
-
-    - AWS credentials configured (via environment variables, AWS CLI, or instance metadata)
-    - A deployed SageMaker endpoint with Deepgram model: https://developers.deepgram.com/docs/deploy-amazon-sagemaker
-    - Deepgram SDK for LiveOptions configuration
-
-    Example::
-
-        stt = DeepgramSageMakerSTTService(
-            endpoint_name="my-deepgram-endpoint",
-            region="us-east-2",
-            live_options=LiveOptions(
-                model="nova-3",
-                language="en",
-                interim_results=True,
-                punctuate=True,
-            ),
-        )
-    """
-
-    def __init__(
-        self,
-        *,
-        endpoint_name: str,
-        region: str,
-        sample_rate: Optional[int] = None,
-        live_options: Optional[LiveOptions] = None,
-        **kwargs,
-    ):
-        """Initialize the Deepgram SageMaker STT service.
-
-        Args:
-            endpoint_name: Name of the SageMaker endpoint with Deepgram model
-                deployed (e.g., "my-deepgram-nova-3-endpoint").
-            region: AWS region where the endpoint is deployed (e.g., "us-east-2").
-            sample_rate: Audio sample rate in Hz. If None, uses value from
-                live_options or defaults to the value from StartFrame.
-            live_options: Deepgram LiveOptions for detailed configuration. If None,
-                uses sensible defaults (nova-3 model, English, interim results enabled).
-            **kwargs: Additional arguments passed to the parent STTService.
-        """
-        sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
-        super().__init__(sample_rate=sample_rate, **kwargs)
-
-        self._endpoint_name = endpoint_name
-        self._region = region
-
-        # Create default options similar to DeepgramSTTService
-        default_options = LiveOptions(
-            encoding="linear16",
-            language=Language.EN,
-            model="nova-3",
-            channels=1,
-            interim_results=True,
-            punctuate=True,
-        )
-
-        # Merge with provided options
-        merged_options = default_options.to_dict()
-        if live_options:
-            default_model = default_options.model
-            merged_options.update(live_options.to_dict())
-            # Handle the "None" string bug from deepgram-sdk
-            if "model" in merged_options and merged_options["model"] == "None":
-                merged_options["model"] = default_model
-
-        # Convert Language enum to string if needed
-        if "language" in merged_options and isinstance(merged_options["language"], Language):
-            merged_options["language"] = merged_options["language"].value
-
-        self.set_model_name(merged_options["model"])
-        self._settings = merged_options
-
-        self._client: Optional[SageMakerBidiClient] = None
-        self._response_task: Optional[asyncio.Task] = None
-        self._keepalive_task: Optional[asyncio.Task] = None
-
-    def can_generate_metrics(self) -> bool:
-        """Check if this service can generate processing metrics.
-
-        Returns:
-            True, as Deepgram SageMaker service supports metrics generation.
-        """
-        return True
-
-    async def set_model(self, model: str):
-        """Set the Deepgram model and reconnect.
-
-        Disconnects from the current session, updates the model setting, and
-        establishes a new connection with the updated model.
-
-        Args:
-            model: The Deepgram model name to use (e.g., "nova-3").
-        """
-        await super().set_model(model)
-        logger.info(f"Switching STT model to: [{model}]")
-        self._settings["model"] = model
-        await self._disconnect()
-        await self._connect()
-
-    async def set_language(self, language: Language):
-        """Set the recognition language and reconnect.
-
-        Disconnects from the current session, updates the language setting, and
-        establishes a new connection with the updated language.
-
-        Args:
-            language: The language to use for speech recognition (e.g., Language.EN,
-                Language.ES).
-        """
-        logger.info(f"Switching STT language to: [{language}]")
-        self._settings["language"] = language
-        await self._disconnect()
-        await self._connect()
-
-    async def start(self, frame: StartFrame):
-        """Start the Deepgram SageMaker STT service.
-
-        Args:
-            frame: The start frame containing initialization parameters.
-        """
-        await super().start(frame)
-        self._settings["sample_rate"] = self.sample_rate
-        await self._connect()
-
-    async def stop(self, frame: EndFrame):
-        """Stop the Deepgram SageMaker STT service.
-
-        Args:
-            frame: The end frame.
-        """
-        await super().stop(frame)
-        await self._disconnect()
-
-    async def cancel(self, frame: CancelFrame):
-        """Cancel the Deepgram SageMaker STT service.
-
-        Args:
-            frame: The cancel frame.
-        """
-        await super().cancel(frame)
-        await self._disconnect()
-
-    async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
-        """Send audio data to Deepgram for transcription.
-
-        Args:
-            audio: Raw audio bytes to transcribe.
-
-        Yields:
-            Frame: None (transcription results come via BiDi stream callbacks).
-        """
-        if self._client and self._client.is_active:
-            try:
-                await self._client.send_audio_chunk(audio)
-            except Exception as e:
-                yield ErrorFrame(error=f"Unknown error occurred: {e}")
-        yield None
-
-    async def _connect(self):
-        """Connect to the SageMaker endpoint and start the BiDi session.
-
-        Builds the Deepgram query string from settings, creates the BiDi client,
-        starts the streaming session, and launches background tasks for processing
-        responses and sending KeepAlive messages.
-        """
-        logger.debug("Connecting to Deepgram on SageMaker...")
-
-        # Update sample rate in settings
-        self._settings["sample_rate"] = self.sample_rate
-
-        # Build query string from settings, converting booleans to strings
-        query_params = {}
-        for key, value in self._settings.items():
-            if value is not None:
-                # Convert boolean values to lowercase strings for Deepgram API
-                if isinstance(value, bool):
-                    query_params[key] = str(value).lower()
-                else:
-                    query_params[key] = str(value)
-
-        query_string = "&".join(f"{k}={v}" for k, v in query_params.items())
-
-        # Create BiDi client
-        self._client = SageMakerBidiClient(
-            endpoint_name=self._endpoint_name,
-            region=self._region,
-            model_invocation_path="v1/listen",
-            model_query_string=query_string,
-        )
-
-        try:
-            # Start the session
-            await self._client.start_session()
-
-            # Start processing responses in the background
-            self._response_task = self.create_task(self._process_responses())
-
-            # Start keepalive task to maintain connection
-            self._keepalive_task = self.create_task(self._send_keepalive())
-
-            logger.debug("Connected to Deepgram on SageMaker")
-            await self._call_event_handler("on_connected")
-
-        except Exception as e:
-            await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
-            await self._call_event_handler("on_connection_error", str(e))
-
-    async def _disconnect(self):
-        """Disconnect from the SageMaker endpoint.
-
-        Sends a CloseStream message to Deepgram, cancels background tasks
-        (KeepAlive and response processing), and closes the BiDi session.
-        Safe to call multiple times.
-        """
-        if self._client and self._client.is_active:
-            logger.debug("Disconnecting from Deepgram on SageMaker...")
-
-            # Send CloseStream message to Deepgram
-            try:
-                await self._client.send_json({"type": "CloseStream"})
-            except Exception as e:
-                logger.warning(f"Failed to send CloseStream message: {e}")
-
-            # Cancel keepalive task
-            if self._keepalive_task and not self._keepalive_task.done():
-                await self.cancel_task(self._keepalive_task)
-
-            # Cancel response processing task
-            if self._response_task and not self._response_task.done():
-                await self.cancel_task(self._response_task)
-
-            # Close the BiDi session
-            await self._client.close_session()
-
-            logger.debug("Disconnected from Deepgram on SageMaker")
-            await self._call_event_handler("on_disconnected")
-
-    async def _send_keepalive(self):
-        """Send periodic KeepAlive messages to maintain the connection.
-
-        Sends a KeepAlive JSON message to Deepgram every 5 seconds while the
-        connection is active. This prevents the connection from timing out during
-        periods of silence.
-        """
-        while self._client and self._client.is_active:
-            await asyncio.sleep(5)
-            if self._client and self._client.is_active:
-                try:
-                    await self._client.send_json({"type": "KeepAlive"})
-                except Exception as e:
-                    logger.warning(f"Failed to send KeepAlive: {e}")
-
-    async def _process_responses(self):
-        """Process streaming responses from Deepgram on SageMaker.
-
-        Continuously receives responses from the BiDi stream, decodes the payload,
-        parses JSON responses from Deepgram, and processes transcription results.
-        Runs as a background task until the connection is closed or cancelled.
-        """
-        try:
-            while self._client and self._client.is_active:
-                result = await self._client.receive_response()
-
-                if result is None:
-                    break
-
-                # Check if this is a PayloadPart with bytes
-                if hasattr(result, "value") and hasattr(result.value, "bytes_"):
-                    if result.value.bytes_:
-                        response_data = result.value.bytes_.decode("utf-8")
-
-                        try:
-                            # Parse JSON response from Deepgram
-                            parsed = json.loads(response_data)
-
-                            # Extract and process transcript if available
-                            if "channel" in parsed:
-                                await self._handle_transcript_response(parsed)
-
-                        except json.JSONDecodeError:
-                            logger.warning(f"Non-JSON response: {response_data}")
-
-        except asyncio.CancelledError:
-            logger.debug("Response processor cancelled")
-        except Exception as e:
-            await self.push_error(error_msg=f"Unknown error occurred: {e}", exception=e)
-        finally:
-            logger.debug("Response processor stopped")
-
-    async def _handle_transcript_response(self, parsed: dict):
-        """Handle a transcript response from Deepgram.
-
-        Extracts the transcript text, determines if it's final or interim, extracts
-        language information, and pushes the appropriate frame (TranscriptionFrame
-        or InterimTranscriptionFrame) downstream.
-
-        Args:
-            parsed: The parsed JSON response from Deepgram containing channel,
-                alternatives, transcript, and metadata.
-        """
-        alternatives = parsed.get("channel", {}).get("alternatives", [])
-        if not alternatives or not alternatives[0].get("transcript"):
-            return
-
-        transcript = alternatives[0]["transcript"]
-        if not transcript.strip():
-            return
-
-        # Stop TTFB metrics on first transcript
-        await self.stop_ttfb_metrics()
-
-        is_final = parsed.get("is_final", False)
-        speech_final = parsed.get("speech_final", False)
-
-        # Extract language if available
-        language = None
-        if alternatives[0].get("languages"):
-            language = alternatives[0]["languages"][0]
-            language = Language(language)
-
-        if is_final and speech_final:
-            # Final transcription
-            await self.push_frame(
-                TranscriptionFrame(
-                    transcript,
-                    self._user_id,
-                    time_now_iso8601(),
-                    language,
-                    result=parsed,
-                )
-            )
-            await self._handle_transcription(transcript, is_final, language)
-            await self.stop_processing_metrics()
-        else:
-            # Interim transcription
-            await self.push_frame(
-                InterimTranscriptionFrame(
-                    transcript,
-                    self._user_id,
-                    time_now_iso8601(),
-                    language,
-                    result=parsed,
-                )
-            )
-
-    @traced_stt
-    async def _handle_transcription(
-        self, transcript: str, is_final: bool, language: Optional[Language] = None
-    ):
-        """Handle a transcription result with tracing.
-
-        This method is decorated with @traced_stt for observability and tracing
-        integration. The actual transcription processing is handled by the parent
-        class and observers.
-
-        Args:
-            transcript: The transcribed text.
-            is_final: Whether this is a final transcription result.
-            language: The detected language of the transcription, if available.
-        """
-        pass
-
-    async def start_metrics(self):
-        """Start TTFB and processing metrics collection."""
-        await self.start_ttfb_metrics()
-        await self.start_processing_metrics()
-
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        """Process frames with Deepgram SageMaker-specific handling.
-
-        Args:
-            frame: The frame to process.
-            direction: The direction of frame processing.
-        """
-        await super().process_frame(frame, direction)
-
-        # Start metrics when user starts speaking (if VAD is not provided by Deepgram)
-        if isinstance(frame, VADUserStartedSpeakingFrame):
-            await self.start_metrics()
-        elif isinstance(frame, VADUserStoppedSpeakingFrame):
-            # Send finalize message to Deepgram when user stops speaking
-            # This tells Deepgram to flush any remaining audio and return final results
-            if self._client and self._client.is_active:
-                try:
-                    await self._client.send_json({"type": "Finalize"})
-                except Exception as e:
-                    logger.warning(f"Error sending Finalize message: {e}")
+from pipecat.services.deepgram.sagemaker.stt import *  # noqa: E402, F401, F403

src/pipecat/services/deepgram/tts.py

@@ -11,7 +11,8 @@ for generating speech from text using various voice models.
 """
 
 import json
-from typing import AsyncGenerator, Optional
+from dataclasses import dataclass
+from typing import Any, AsyncGenerator, Optional
 
 import aiohttp
 from loguru import logger
@@ -21,14 +22,11 @@ from pipecat.frames.frames import (
     EndFrame,
     ErrorFrame,
     Frame,
-    InterruptionFrame,
-    LLMFullResponseEndFrame,
     StartFrame,
     TTSAudioRawFrame,
-    TTSStartedFrame,
     TTSStoppedFrame,
 )
-from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.settings import TTSSettings
 from pipecat.services.tts_service import TTSService, WebsocketTTSService
 from pipecat.utils.tracing.service_decorators import traced_tts
 
@@ -43,6 +41,13 @@ except ModuleNotFoundError as e:
     raise Exception(f"Missing module: {e}")
 
 
+@dataclass
+class DeepgramTTSSettings(TTSSettings):
+    """Settings for DeepgramTTSService and DeepgramHttpTTSService."""
+
+    pass
+
+
 class DeepgramTTSService(WebsocketTTSService):
     """Deepgram WebSocket-based text-to-speech service.
 
@@ -51,26 +56,36 @@ class DeepgramTTSService(WebsocketTTSService):
     message for conversational AI use cases.
     """
 
+    Settings = DeepgramTTSSettings
+    _settings: Settings
+
     SUPPORTED_ENCODINGS = ("linear16", "mulaw", "alaw")
 
     def __init__(
         self,
         *,
         api_key: str,
-        voice: str = "aura-2-helena-en",
+        voice: Optional[str] = None,
         base_url: str = "wss://api.deepgram.com",
         sample_rate: Optional[int] = None,
         encoding: str = "linear16",
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Deepgram WebSocket TTS service.
 
         Args:
             api_key: Deepgram API key for authentication.
-            voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en".
+            voice: Voice model to use for synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepgramTTSService.Settings(voice=...)`` instead.
+
             base_url: WebSocket base URL for Deepgram API. Defaults to "wss://api.deepgram.com".
             sample_rate: Audio sample rate in Hz. If None, uses service default.
             encoding: Audio encoding format. Defaults to "linear16". Must be one of SUPPORTED_ENCODINGS.
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional arguments passed to parent InterruptibleTTSService class.
 
         Raises:
@@ -81,19 +96,38 @@ class DeepgramTTSService(WebsocketTTSService):
                 f"Unsupported encoding '{encoding}'. Must be one of {', '.join(self.SUPPORTED_ENCODINGS)} for WebSocket TTS."
             )
 
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            voice="aura-2-helena-en",
+            language=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice is not None:
+            self._warn_init_param_moved_to_settings("voice", "voice")
+            default_settings.model = voice
+            default_settings.voice = voice
+
+        # 3. (No step 3, as there's no params object to apply)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
         super().__init__(
             sample_rate=sample_rate,
             pause_frame_processing=True,
-            push_stop_frames=True,
+            push_stop_frames=False,
+            push_start_frame=True,
+            append_trailing_space=True,
+            settings=default_settings,
             **kwargs,
         )
 
         self._api_key = api_key
         self._base_url = base_url
-        self._settings = {
-            "encoding": encoding,
-        }
-        self.set_voice(voice)
+        self._encoding = encoding
 
         self._receive_task = None
 
@@ -132,21 +166,10 @@ class DeepgramTTSService(WebsocketTTSService):
         await super().cancel(frame)
         await self._disconnect()
 
-    async def process_frame(self, frame: Frame, direction: FrameDirection):
-        """Process frames with special handling for LLM response end.
-
-        Args:
-            frame: The frame to process.
-            direction: The direction of frame processing.
-        """
-        await super().process_frame(frame, direction)
-
-        # When the LLM finishes responding, flush any remaining text in Deepgram's buffer
-        if isinstance(frame, (LLMFullResponseEndFrame, EndFrame)):
-            await self.flush_audio()
-
     async def _connect(self):
         """Connect to Deepgram WebSocket and start receive task."""
+        await super()._connect()
+
         await self._connect_websocket()
 
         if self._websocket and not self._receive_task:
@@ -154,12 +177,36 @@ class DeepgramTTSService(WebsocketTTSService):
 
     async def _disconnect(self):
         """Disconnect from Deepgram WebSocket and clean up tasks."""
+        await super()._disconnect()
+
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
 
         await self._disconnect_websocket()
 
+    async def _update_settings(self, delta: TTSSettings) -> dict[str, Any]:
+        """Apply a settings delta.
+
+        Args:
+            delta: A :class:`TTSSettings` (or ``DeepgramTTSService.Settings``) delta.
+
+        Returns:
+            Dict mapping changed field names to their previous values.
+        """
+        changed = await super()._update_settings(delta)
+
+        # Deepgram uses voice as the model, so keep them in sync for metrics
+        if "voice" in changed:
+            self._settings.model = self._settings.voice
+            self._sync_model_name_to_metrics()
+
+        if changed:
+            await self._disconnect()
+            await self._connect()
+
+        return changed
+
     async def _connect_websocket(self):
         """Connect to Deepgram WebSocket API with configured settings."""
         try:
@@ -170,8 +217,8 @@ class DeepgramTTSService(WebsocketTTSService):
 
             # Build WebSocket URL with query parameters
             params = []
-            params.append(f"model={self._voice_id}")
-            params.append(f"encoding={self._settings['encoding']}")
+            params.append(f"model={self._settings.voice}")
+            params.append(f"encoding={self._encoding}")
             params.append(f"sample_rate={self.sample_rate}")
 
             url = f"{self._base_url}/v1/speak?{'&'.join(params)}"
@@ -215,19 +262,19 @@ class DeepgramTTSService(WebsocketTTSService):
             return self._websocket
         raise Exception("Websocket not connected")
 
-    async def _handle_interruption(self, frame: InterruptionFrame, direction: FrameDirection):
-        """Handle interruption by sending Clear message to Deepgram.
+    async def on_audio_context_interrupted(self, context_id: str):
+        """Send Clear message to Deepgram when an audio context is interrupted.
 
         The Clear message will clear Deepgram's internal text buffer and stop
         sending audio, allowing for a new response to be generated.
-        """
-        await super()._handle_interruption(frame, direction)
 
-        # Send Clear message to stop current audio generation
+        Args:
+            context_id: The ID of the audio context that was interrupted.
+        """
+        await self.stop_all_metrics()
         if self._websocket:
             try:
-                clear_msg = {"type": "Clear"}
-                await self._websocket.send(json.dumps(clear_msg))
+                await self._websocket.send(json.dumps({"type": "Clear"}))
             except Exception as e:
                 logger.error(f"{self} error sending Clear message: {e}")
 
@@ -236,9 +283,9 @@ class DeepgramTTSService(WebsocketTTSService):
         async for message in self._get_websocket():
             if isinstance(message, bytes):
                 # Binary message contains audio data
-                await self.stop_ttfb_metrics()
-                frame = TTSAudioRawFrame(message, self.sample_rate, 1)
-                await self.push_frame(frame)
+                ctx_id = self.get_active_audio_context_id()
+                frame = TTSAudioRawFrame(message, self.sample_rate, 1, context_id=ctx_id)
+                await self.append_to_audio_context(ctx_id, frame)
             elif isinstance(message, str):
                 # Text message contains metadata or control messages
                 try:
@@ -249,12 +296,15 @@ class DeepgramTTSService(WebsocketTTSService):
                         logger.trace(f"Received metadata: {msg}")
                     elif msg_type == "Flushed":
                         logger.trace(f"Received Flushed: {msg}")
-                        # Flushed indicates the end of audio generation for the current buffer
-                        # This happens after flush_audio() is called
+                        ctx_id = self.get_active_audio_context_id()
+                        await self.append_to_audio_context(
+                            ctx_id, TTSStoppedFrame(context_id=ctx_id)
+                        )
+                        await self.remove_audio_context(ctx_id)
                     elif msg_type == "Cleared":
                         logger.trace(f"Received Cleared: {msg}")
-                        # Buffer has been cleared after interruption
-                        # TTSStoppedFrame will be sent by the interruption handler
+                        # Buffer has been cleared after interruption.
+                        # The on_audio_context_interrupted handler already cleaned up.
                     elif msg_type == "Warning":
                         logger.warning(
                             f"{self} warning: {msg.get('description', 'Unknown warning')}"
@@ -264,7 +314,7 @@ class DeepgramTTSService(WebsocketTTSService):
                 except json.JSONDecodeError:
                     logger.error(f"Invalid JSON message: {message}")
 
-    async def flush_audio(self):
+    async def flush_audio(self, context_id: Optional[str] = None):
         """Flush any pending audio synthesis by sending Flush command.
 
         This should be called when the LLM finishes a complete response to force
@@ -278,33 +328,27 @@ class DeepgramTTSService(WebsocketTTSService):
                 logger.error(f"{self} error sending Flush message: {e}")
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Deepgram's WebSocket TTS API.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the synthesized speech, plus start/stop frames.
         """
-        # Append trailing space to prevent TTS from vocalizing trailing periods as "dot"
-        text_with_trailing_space = text + " "
-        logger.debug(f"{self}: Generating TTS [{text_with_trailing_space}]")
+        logger.debug(f"{self}: Generating TTS [{text}]")
 
         try:
             # Reconnect if the websocket is closed
             if not self._websocket or self._websocket.state is State.CLOSED:
                 await self._connect()
 
-            await self.start_ttfb_metrics()
-            await self.start_tts_usage_metrics(text_with_trailing_space)
-
-            yield TTSStartedFrame()
-
             # Send text message to Deepgram
             # Note: We don't send Flush here - that should only be sent when the
             # LLM finishes a complete response via flush_audio()
-            speak_msg = {"type": "Speak", "text": text_with_trailing_space}
+            speak_msg = {"type": "Speak", "text": text}
             await self._get_websocket().send(json.dumps(speak_msg))
 
             # The audio frames will be handled in _receive_messages
@@ -322,37 +366,69 @@ class DeepgramHttpTTSService(TTSService):
     configurable sample rates and quality settings.
     """
 
+    Settings = DeepgramTTSSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
         api_key: str,
-        voice: str = "aura-2-helena-en",
+        voice: Optional[str] = None,
         aiohttp_session: aiohttp.ClientSession,
         base_url: str = "https://api.deepgram.com",
         sample_rate: Optional[int] = None,
         encoding: str = "linear16",
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the Deepgram TTS service.
 
         Args:
             api_key: Deepgram API key for authentication.
-            voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en".
+            voice: Voice model to use for synthesis.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepgramHttpTTSService.Settings(voice=...)`` instead.
+
             aiohttp_session: Shared aiohttp session for HTTP requests with connection pooling.
             base_url: Custom base URL for Deepgram API. Defaults to "https://api.deepgram.com".
             sample_rate: Audio sample rate in Hz. If None, uses service default.
             encoding: Audio encoding format. Defaults to "linear16".
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional arguments passed to parent TTSService class.
         """
-        super().__init__(sample_rate=sample_rate, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model=None,
+            voice="aura-2-helena-en",
+            language=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if voice is not None:
+            self._warn_init_param_moved_to_settings("voice", "voice")
+            default_settings.model = voice
+            default_settings.voice = voice
+
+        # 3. (No step 3, as there's no params object to apply)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            push_start_frame=True,
+            push_stop_frames=True,
+            settings=default_settings,
+            **kwargs,
+        )
 
         self._api_key = api_key
         self._session = aiohttp_session
         self._base_url = base_url
-        self._settings = {
-            "encoding": encoding,
-        }
-        self.set_voice(voice)
+        self._encoding = encoding
 
     def can_generate_metrics(self) -> bool:
         """Check if the service can generate metrics.
@@ -363,11 +439,12 @@ class DeepgramHttpTTSService(TTSService):
         return True
 
     @traced_tts
-    async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
+    async def run_tts(self, text: str, context_id: str) -> AsyncGenerator[Frame, None]:
         """Generate speech from text using Deepgram's TTS API.
 
         Args:
             text: The text to synthesize into speech.
+            context_id: The context ID for tracking audio frames.
 
         Yields:
             Frame: Audio frames containing the synthesized speech, plus start/stop frames.
@@ -380,8 +457,8 @@ class DeepgramHttpTTSService(TTSService):
         headers = {"Authorization": f"Token {self._api_key}", "Content-Type": "application/json"}
 
         params = {
-            "model": self._voice_id,
-            "encoding": self._settings["encoding"],
+            "model": self._settings.voice,
+            "encoding": self._encoding,
             "sample_rate": self.sample_rate,
             "container": "none",
         }
@@ -401,7 +478,6 @@ class DeepgramHttpTTSService(TTSService):
                     raise Exception(f"HTTP {response.status}: {error_text}")
 
                 await self.start_tts_usage_metrics(text)
-                yield TTSStartedFrame()
 
                 CHUNK_SIZE = self.chunk_size
 
@@ -416,9 +492,8 @@ class DeepgramHttpTTSService(TTSService):
                             audio=chunk,
                             sample_rate=self.sample_rate,
                             num_channels=1,
+                            context_id=context_id,
                         )
 
-            yield TTSStoppedFrame()
-
         except Exception as e:
             yield ErrorFrame(f"Error getting audio: {str(e)}")

src/pipecat/services/deepgram/tts_sagemaker.py

@@ -0,0 +1,18 @@
+#
+# Copyright (c) 2024-2026, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""Deprecated: use ``pipecat.services.deepgram.sagemaker.tts`` instead."""
+
+import warnings
+
+warnings.warn(
+    "Module `pipecat.services.deepgram.tts_sagemaker` is deprecated, "
+    "use `pipecat.services.deepgram.sagemaker.tts` instead.",
+    DeprecationWarning,
+    stacklevel=2,
+)
+
+from pipecat.services.deepgram.sagemaker.tts import *  # noqa: E402, F401, F403

src/pipecat/services/deepseek/llm.py

@@ -6,14 +6,23 @@
 
 """DeepSeek LLM service implementation using OpenAI-compatible interface."""
 
-from typing import List
+from dataclasses import dataclass
+from typing import Optional
 
 from loguru import logger
 
 from pipecat.adapters.services.open_ai_adapter import OpenAILLMInvocationParams
+from pipecat.services.openai.base_llm import BaseOpenAILLMService
 from pipecat.services.openai.llm import OpenAILLMService
 
 
+@dataclass
+class DeepSeekLLMSettings(BaseOpenAILLMService.Settings):
+    """Settings for DeepSeekLLMService."""
+
+    pass
+
+
 class DeepSeekLLMService(OpenAILLMService):
     """A service for interacting with DeepSeek's API using the OpenAI-compatible interface.
 
@@ -21,12 +30,16 @@ class DeepSeekLLMService(OpenAILLMService):
     maintaining full compatibility with OpenAI's interface and functionality.
     """
 
+    Settings = DeepSeekLLMSettings
+    _settings: Settings
+
     def __init__(
         self,
         *,
         api_key: str,
         base_url: str = "https://api.deepseek.com/v1",
-        model: str = "deepseek-chat",
+        model: Optional[str] = None,
+        settings: Optional[Settings] = None,
         **kwargs,
     ):
         """Initialize the DeepSeek LLM service.
@@ -35,9 +48,29 @@ class DeepSeekLLMService(OpenAILLMService):
             api_key: The API key for accessing DeepSeek's API.
             base_url: The base URL for DeepSeek API. Defaults to "https://api.deepseek.com/v1".
             model: The model identifier to use. Defaults to "deepseek-chat".
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=DeepSeekLLMService.Settings(model=...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
             **kwargs: Additional keyword arguments passed to OpenAILLMService.
         """
-        super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(model="deepseek-chat")
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. (No step 3, as there's no params object to apply)
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(api_key=api_key, base_url=base_url, settings=default_settings, **kwargs)
 
     def create_client(self, api_key=None, base_url=None, **kwargs):
         """Create OpenAI-compatible client for DeepSeek API endpoint.
@@ -67,18 +100,18 @@ class DeepSeekLLMService(OpenAILLMService):
             Dictionary of parameters for the chat completion request.
         """
         params = {
-            "model": self.model_name,
+            "model": self._settings.model,
             "stream": True,
             "stream_options": {"include_usage": True},
-            "frequency_penalty": self._settings["frequency_penalty"],
-            "presence_penalty": self._settings["presence_penalty"],
-            "temperature": self._settings["temperature"],
-            "top_p": self._settings["top_p"],
-            "max_tokens": self._settings["max_tokens"],
+            "frequency_penalty": self._settings.frequency_penalty,
+            "presence_penalty": self._settings.presence_penalty,
+            "temperature": self._settings.temperature,
+            "top_p": self._settings.top_p,
+            "max_tokens": self._settings.max_tokens,
         }
 
         # Messages, tools, tool_choice
         params.update(params_from_context)
 
-        params.update(self._settings["extra"])
+        params.update(self._settings.extra)
         return params

src/pipecat/services/elevenlabs/stt.py

@@ -11,11 +11,13 @@ using segmented audio processing. The service uploads audio files and receives
 transcription results directly.
 """
 
+import asyncio
 import base64
 import io
 import json
+from dataclasses import dataclass, field
 from enum import Enum
-from typing import AsyncGenerator, Optional
+from typing import Any, AsyncGenerator, Optional
 
 import aiohttp
 from loguru import logger
@@ -33,6 +35,8 @@ from pipecat.frames.frames import (
     VADUserStoppedSpeakingFrame,
 )
 from pipecat.processors.frame_processor import FrameDirection
+from pipecat.services.settings import NOT_GIVEN, STTSettings, _NotGiven
+from pipecat.services.stt_latency import ELEVENLABS_REALTIME_TTFS_P99, ELEVENLABS_TTFS_P99
 from pipecat.services.stt_service import SegmentedSTTService, WebsocketSTTService
 from pipecat.transcriptions.language import Language, resolve_language
 from pipecat.utils.time import time_now_iso8601
@@ -166,6 +170,44 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]:
     return resolve_language(language, LANGUAGE_MAP, use_base_code=False)
 
 
+class CommitStrategy(str, Enum):
+    """Commit strategies for transcript segmentation."""
+
+    MANUAL = "manual"
+    VAD = "vad"
+
+
+@dataclass
+class ElevenLabsSTTSettings(STTSettings):
+    """Settings for ElevenLabsSTTService.
+
+    Parameters:
+        tag_audio_events: Whether to include audio events like (laughter),
+            (coughing) in the transcription.
+    """
+
+    tag_audio_events: bool | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
+@dataclass
+class ElevenLabsRealtimeSTTSettings(STTSettings):
+    """Settings for ElevenLabsRealtimeSTTService.
+
+    See ``ElevenLabsRealtimeSTTService.InputParams`` for detailed descriptions.
+
+    Parameters:
+        vad_silence_threshold_secs: Seconds of silence before VAD commits (0.3-3.0).
+        vad_threshold: VAD sensitivity (0.1-0.9, lower is more sensitive).
+        min_speech_duration_ms: Minimum speech duration for VAD (50-2000ms).
+        min_silence_duration_ms: Minimum silence duration for VAD (50-2000ms).
+    """
+
+    vad_silence_threshold_secs: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    vad_threshold: float | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    min_speech_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+    min_silence_duration_ms: int | None | _NotGiven = field(default_factory=lambda: NOT_GIVEN)
+
+
 class ElevenLabsSTTService(SegmentedSTTService):
     """Speech-to-text service using ElevenLabs' file-based API.
 
@@ -174,9 +216,15 @@ class ElevenLabsSTTService(SegmentedSTTService):
     The service uploads audio files to ElevenLabs and receives transcription results directly.
     """
 
+    Settings = ElevenLabsSTTSettings
+    _settings: Settings
+
     class InputParams(BaseModel):
         """Configuration parameters for ElevenLabs STT API.
 
+        .. deprecated:: 0.0.105
+            Use ``settings=ElevenLabsSTTService.Settings(...)`` instead.
+
         Parameters:
             language: Target language for transcription.
             tag_audio_events: Whether to include audio events like (laughter), (coughing), in the transcription.
@@ -191,9 +239,11 @@ class ElevenLabsSTTService(SegmentedSTTService):
         api_key: str,
         aiohttp_session: aiohttp.ClientSession,
         base_url: str = "https://api.elevenlabs.io",
-        model: str = "scribe_v1",
+        model: Optional[str] = None,
         sample_rate: Optional[int] = None,
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = ELEVENLABS_TTFS_P99,
         **kwargs,
     ):
         """Initialize the ElevenLabs STT service.
@@ -202,29 +252,57 @@ class ElevenLabsSTTService(SegmentedSTTService):
             api_key: ElevenLabs API key for authentication.
             aiohttp_session: aiohttp ClientSession for HTTP requests.
             base_url: Base URL for ElevenLabs API.
-            model: Model ID for transcription. Defaults to "scribe_v1".
+            model: Model ID for transcription.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=ElevenLabsSTTService.Settings(model=...)`` instead.
+
             sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
             params: Configuration parameters for the STT service.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=ElevenLabsSTTService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
             **kwargs: Additional arguments passed to SegmentedSTTService.
         """
-        super().__init__(
-            sample_rate=sample_rate,
-            **kwargs,
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="scribe_v2",
+            language=Language.EN,
+            tag_audio_events=None,
         )
 
-        params = params or ElevenLabsSTTService.InputParams()
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                if params.language is not None:
+                    default_settings.language = params.language
+                default_settings.tag_audio_events = params.tag_audio_events
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
+        super().__init__(
+            sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            settings=default_settings,
+            **kwargs,
+        )
 
         self._api_key = api_key
         self._base_url = base_url
         self._session = aiohttp_session
-        self._model_id = model
-        self._tag_audio_events = params.tag_audio_events
-
-        self._settings = {
-            "language": self.language_to_service_language(params.language)
-            if params.language
-            else "eng",
-        }
 
     def can_generate_metrics(self) -> bool:
         """Check if the service can generate processing metrics.
@@ -245,28 +323,6 @@ class ElevenLabsSTTService(SegmentedSTTService):
         """
         return language_to_elevenlabs_language(language)
 
-    async def set_language(self, language: Language):
-        """Set the transcription language.
-
-        Args:
-            language: The language to use for speech-to-text transcription.
-        """
-        logger.info(f"Switching STT language to: [{language}]")
-        self._settings["language"] = self.language_to_service_language(language)
-
-    async def set_model(self, model: str):
-        """Set the STT model.
-
-        Args:
-            model: The model name to use for transcription.
-
-        Note:
-            ElevenLabs STT API does not currently support model selection.
-            This method is provided for interface compatibility.
-        """
-        await super().set_model(model)
-        logger.info(f"Model setting [{model}] noted, but ElevenLabs STT uses default model")
-
     async def _transcribe_audio(self, audio_data: bytes) -> dict:
         """Upload audio data to ElevenLabs and get transcription result.
 
@@ -291,10 +347,11 @@ class ElevenLabsSTTService(SegmentedSTTService):
             content_type="audio/x-wav",
         )
 
-        # Add required model_id, language_code, and tag_audio_events
-        data.add_field("model_id", self._model_id)
-        data.add_field("language_code", self._settings["language"])
-        data.add_field("tag_audio_events", str(self._tag_audio_events).lower())
+        # Add required model_id and language_code
+        data.add_field("model_id", self._settings.model)
+        data.add_field("language_code", self._settings.language)
+        if self._settings.tag_audio_events is not None:
+            data.add_field("tag_audio_events", str(self._settings.tag_audio_events).lower())
 
         async with self._session.post(url, data=data, headers=headers) as response:
             if response.status != 200:
@@ -310,7 +367,6 @@ class ElevenLabsSTTService(SegmentedSTTService):
         self, transcript: str, is_final: bool, language: Optional[str] = None
     ):
         """Handle a transcription result with tracing."""
-        await self.stop_ttfb_metrics()
         await self.stop_processing_metrics()
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
@@ -328,7 +384,6 @@ class ElevenLabsSTTService(SegmentedSTTService):
         """
         try:
             await self.start_processing_metrics()
-            await self.start_ttfb_metrics()
 
             # Upload audio and get transcription result directly
             result = await self._transcribe_audio(audio)
@@ -382,13 +437,6 @@ def audio_format_from_sample_rate(sample_rate: int) -> str:
     return "pcm_16000"
 
 
-class CommitStrategy(str, Enum):
-    """Commit strategies for transcript segmentation."""
-
-    MANUAL = "manual"
-    VAD = "vad"
-
-
 class ElevenLabsRealtimeSTTService(WebsocketSTTService):
     """Speech-to-text service using ElevenLabs' Realtime WebSocket API.
 
@@ -401,9 +449,15 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
     commit transcript segments, providing consistency with other STT services.
     """
 
+    Settings = ElevenLabsRealtimeSTTSettings
+    _settings: Settings
+
     class InputParams(BaseModel):
         """Configuration parameters for ElevenLabs Realtime STT API.
 
+        .. deprecated:: 0.0.105
+            Use ``settings=ElevenLabsRealtimeSTTService.Settings(...)`` instead.
+
         Parameters:
             language_code: ISO-639-1 or ISO-639-3 language code. Leave None for auto-detection.
             commit_strategy: How to segment speech - manual (Pipecat VAD) or vad (ElevenLabs VAD).
@@ -435,9 +489,15 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         *,
         api_key: str,
         base_url: str = "api.elevenlabs.io",
-        model: str = "scribe_v2_realtime",
+        commit_strategy: CommitStrategy = CommitStrategy.MANUAL,
+        model: Optional[str] = None,
         sample_rate: Optional[int] = None,
+        include_timestamps: bool = False,
+        enable_logging: bool = False,
+        include_language_detection: bool = False,
         params: Optional[InputParams] = None,
+        settings: Optional[Settings] = None,
+        ttfs_p99_latency: Optional[float] = ELEVENLABS_REALTIME_TTFS_P99,
         **kwargs,
     ):
         """Initialize the ElevenLabs Realtime STT service.
@@ -445,26 +505,85 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         Args:
             api_key: ElevenLabs API key for authentication.
             base_url: Base URL for ElevenLabs WebSocket API.
-            model: Model ID for transcription. Defaults to "scribe_v2_realtime".
+            commit_strategy: How to segment speech — ``CommitStrategy.MANUAL``
+                (Pipecat VAD) or ``CommitStrategy.VAD`` (ElevenLabs VAD).
+                Defaults to ``CommitStrategy.MANUAL``.
+            model: Model ID for transcription.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=ElevenLabsRealtimeSTTService.Settings(model=...)`` instead.
+
             sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
+            include_timestamps: Whether to include word-level timestamps in transcripts.
+            enable_logging: Whether to enable logging on ElevenLabs' side.
+            include_language_detection: Whether to include language detection in transcripts.
             params: Configuration parameters for the STT service.
+
+                .. deprecated:: 0.0.105
+                    Use ``settings=ElevenLabsRealtimeSTTService.Settings(...)`` instead.
+
+            settings: Runtime-updatable settings. When provided alongside deprecated
+                parameters, ``settings`` values take precedence.
+            ttfs_p99_latency: P99 latency from speech end to final transcript in seconds.
+                Override for your deployment. See https://github.com/pipecat-ai/stt-benchmark
             **kwargs: Additional arguments passed to WebsocketSTTService.
         """
+        # 1. Initialize default_settings with hardcoded defaults
+        default_settings = self.Settings(
+            model="scribe_v2_realtime",
+            language=None,
+            vad_silence_threshold_secs=None,
+            vad_threshold=None,
+            min_speech_duration_ms=None,
+            min_silence_duration_ms=None,
+        )
+
+        # 2. Apply direct init arg overrides (deprecated)
+        if model is not None:
+            self._warn_init_param_moved_to_settings("model", "model")
+            default_settings.model = model
+
+        # 3. Apply params overrides — only if settings not provided
+        if params is not None:
+            self._warn_init_param_moved_to_settings("params")
+            if not settings:
+                default_settings.language = params.language_code
+                if params.commit_strategy != CommitStrategy.MANUAL:
+                    commit_strategy = params.commit_strategy
+                default_settings.vad_silence_threshold_secs = params.vad_silence_threshold_secs
+                default_settings.vad_threshold = params.vad_threshold
+                default_settings.min_speech_duration_ms = params.min_speech_duration_ms
+                default_settings.min_silence_duration_ms = params.min_silence_duration_ms
+                include_timestamps = params.include_timestamps
+                enable_logging = params.enable_logging
+                include_language_detection = params.include_language_detection
+
+        # 4. Apply settings delta (canonical API, always wins)
+        if settings is not None:
+            default_settings.apply_update(settings)
+
         super().__init__(
             sample_rate=sample_rate,
+            ttfs_p99_latency=ttfs_p99_latency,
+            keepalive_timeout=10,
+            keepalive_interval=5,
+            settings=default_settings,
             **kwargs,
         )
 
-        params = params or ElevenLabsRealtimeSTTService.InputParams()
-
         self._api_key = api_key
         self._base_url = base_url
-        self._model_id = model
-        self._params = params
         self._audio_format = ""  # initialized in start()
         self._receive_task = None
 
-        self._settings = {"language": params.language_code}
+        # Init-only config (not runtime-updatable).
+        self._commit_strategy = commit_strategy
+        self._include_timestamps = include_timestamps
+        self._enable_logging = enable_logging
+        self._include_language_detection = include_language_detection
+
+        self._connected_event = asyncio.Event()
+        self._connected_event.set()
 
     def can_generate_metrics(self) -> bool:
         """Check if the service can generate processing metrics.
@@ -474,42 +593,25 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         """
         return True
 
-    async def set_language(self, language: Language):
-        """Set the transcription language.
+    async def _update_settings(self, delta: STTSettings) -> dict[str, Any]:
+        """Apply a settings delta and reconnect if anything changed.
 
         Args:
-            language: The language to use for speech-to-text transcription.
+            delta: A :class:`STTSettings` (or ``ElevenLabsRealtimeSTTService.Settings``) delta.
 
-        Note:
-            Changing language requires reconnecting to the WebSocket.
+        Returns:
+            Dict mapping changed field names to their previous values.
         """
-        logger.info(f"Switching STT language to: [{language}]")
-        new_language = (
-            language_to_elevenlabs_language(language)
-            if isinstance(language, Language)
-            else language
-        )
-        self._params.language_code = new_language
-        self._settings["language"] = new_language
-        # Reconnect with new settings
-        await self._disconnect()
-        await self._connect()
+        changed = await super()._update_settings(delta)
 
-    async def set_model(self, model: str):
-        """Set the STT model.
+        if not changed:
+            return changed
 
-        Args:
-            model: The model name to use for transcription.
+        if self._websocket:
+            await self._disconnect()
+            await self._connect()
 
-        Note:
-            Changing model requires reconnecting to the WebSocket.
-        """
-        await super().set_model(model)
-        logger.info(f"Switching STT model to: [{model}]")
-        self._model_id = model
-        # Reconnect with new settings
-        await self._disconnect()
-        await self._connect()
+        return changed
 
     async def start(self, frame: StartFrame):
         """Start the STT service and establish WebSocket connection.
@@ -539,9 +641,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         await super().cancel(frame)
         await self._disconnect()
 
-    async def start_metrics(self):
+    async def _start_metrics(self):
         """Start performance metrics collection for transcription processing."""
-        await self.start_ttfb_metrics()
         await self.start_processing_metrics()
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
@@ -555,10 +656,10 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
 
         if isinstance(frame, VADUserStartedSpeakingFrame):
             # Start metrics when user starts speaking
-            await self.start_metrics()
+            await self._start_metrics()
         elif isinstance(frame, VADUserStoppedSpeakingFrame):
             # Send commit when user stops speaking (manual commit mode)
-            if self._params.commit_strategy == CommitStrategy.MANUAL:
+            if self._commit_strategy == CommitStrategy.MANUAL:
                 if self._websocket and self._websocket.state is State.OPEN:
                     try:
                         commit_message = {
@@ -581,6 +682,9 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         Yields:
             None - transcription results are handled via WebSocket responses.
         """
+        # Wait for any in-flight _connect() to finish before checking state
+        await self._connected_event.wait()
+
         # Reconnect if connection is closed
         if not self._websocket or self._websocket.state is State.CLOSED:
             await self._connect()
@@ -605,19 +709,44 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
 
     async def _connect(self):
         """Establish WebSocket connection to ElevenLabs Realtime STT."""
-        await self._connect_websocket()
+        self._connected_event.clear()
+        try:
+            await self._connect_websocket()
 
-        if self._websocket and not self._receive_task:
-            self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
+            await super()._connect()
+
+            if self._websocket and not self._receive_task:
+                self._receive_task = self.create_task(
+                    self._receive_task_handler(self._report_error)
+                )
+        finally:
+            self._connected_event.set()
 
     async def _disconnect(self):
         """Close WebSocket connection and cleanup tasks."""
+        await super()._disconnect()
+
         if self._receive_task:
             await self.cancel_task(self._receive_task)
             self._receive_task = None
 
         await self._disconnect_websocket()
 
+    async def _send_keepalive(self, silence: bytes):
+        """Send silent audio wrapped in ElevenLabs' JSON protocol.
+
+        Args:
+            silence: Silent 16-bit mono PCM audio bytes.
+        """
+        audio_base64 = base64.b64encode(silence).decode("utf-8")
+        message = {
+            "message_type": "input_audio_chunk",
+            "audio_base_64": audio_base64,
+            "commit": False,
+            "sample_rate": self.sample_rate,
+        }
+        await self._websocket.send(json.dumps(message))
+
     async def _connect_websocket(self):
         """Connect to ElevenLabs Realtime STT WebSocket endpoint."""
         try:
@@ -627,38 +756,40 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
             logger.debug("Connecting to ElevenLabs Realtime STT")
 
             # Build query parameters
-            params = [f"model_id={self._model_id}"]
+            params = [f"model_id={self._settings.model}"]
 
-            if self._params.language_code:
-                params.append(f"language_code={self._params.language_code}")
+            if self._settings.language:
+                params.append(f"language_code={self._settings.language}")
 
             params.append(f"audio_format={self._audio_format}")
-            params.append(f"commit_strategy={self._params.commit_strategy.value}")
+            params.append(f"commit_strategy={self._commit_strategy.value}")
 
             # Add optional parameters
-            if self._params.include_timestamps:
-                params.append(f"include_timestamps={str(self._params.include_timestamps).lower()}")
+            if self._include_timestamps:
+                params.append(f"include_timestamps={str(self._include_timestamps).lower()}")
 
-            if self._params.enable_logging:
-                params.append(f"enable_logging={str(self._params.enable_logging).lower()}")
+            if self._enable_logging:
+                params.append(f"enable_logging={str(self._enable_logging).lower()}")
 
-            if self._params.include_language_detection:
+            if self._include_language_detection:
                 params.append(
-                    f"include_language_detection={str(self._params.include_language_detection).lower()}"
+                    f"include_language_detection={str(self._include_language_detection).lower()}"
                 )
 
             # Add VAD parameters if using VAD commit strategy and values are specified
-            if self._params.commit_strategy == CommitStrategy.VAD:
-                if self._params.vad_silence_threshold_secs is not None:
+            if self._commit_strategy == CommitStrategy.VAD:
+                if self._settings.vad_silence_threshold_secs is not None:
                     params.append(
-                        f"vad_silence_threshold_secs={self._params.vad_silence_threshold_secs}"
+                        f"vad_silence_threshold_secs={self._settings.vad_silence_threshold_secs}"
+                    )
+                if self._settings.vad_threshold is not None:
+                    params.append(f"vad_threshold={self._settings.vad_threshold}")
+                if self._settings.min_speech_duration_ms is not None:
+                    params.append(f"min_speech_duration_ms={self._settings.min_speech_duration_ms}")
+                if self._settings.min_silence_duration_ms is not None:
+                    params.append(
+                        f"min_silence_duration_ms={self._settings.min_silence_duration_ms}"
                     )
-                if self._params.vad_threshold is not None:
-                    params.append(f"vad_threshold={self._params.vad_threshold}")
-                if self._params.min_speech_duration_ms is not None:
-                    params.append(f"min_speech_duration_ms={self._params.min_speech_duration_ms}")
-                if self._params.min_silence_duration_ms is not None:
-                    params.append(f"min_silence_duration_ms={self._params.min_silence_duration_ms}")
 
             ws_url = f"wss://{self._base_url}/v1/speech-to-text/realtime?{'&'.join(params)}"
 
@@ -760,8 +891,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         if not text:
             return
 
-        await self.stop_ttfb_metrics()
-
         # Get language if provided
         language = data.get("language_code")
 
@@ -792,14 +921,13 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         """
         # If timestamps are enabled, skip this message and wait for the
         # committed_transcript_with_timestamps message which contains all the data
-        if self._params.include_timestamps:
+        if self._include_timestamps:
             return
 
         text = data.get("text", "").strip()
         if not text:
             return
 
-        await self.stop_ttfb_metrics()
         await self.stop_processing_metrics()
 
         # Get language if provided
@@ -809,6 +937,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
 
         await self._handle_transcription(text, True, language)
 
+        finalized = self._commit_strategy == CommitStrategy.MANUAL
+
         await self.push_frame(
             TranscriptionFrame(
                 text,
@@ -816,6 +946,7 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
                 time_now_iso8601(),
                 language,
                 result=data,
+                finalized=finalized,
             )
         )
 
@@ -841,7 +972,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
         if not text:
             return
 
-        await self.stop_ttfb_metrics()
         await self.stop_processing_metrics()
 
         # Get language if provided
@@ -851,6 +981,8 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
 
         await self._handle_transcription(text, True, language)
 
+        finalized = self._commit_strategy == CommitStrategy.MANUAL
+
         # This message is sent after committed_transcript when include_timestamps=true.
         # It contains the full transcript data including text and word-level timestamps.
         await self.push_frame(
@@ -860,5 +992,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
                 time_now_iso8601(),
                 language,
                 result=data,
+                finalized=finalized,
             )
         )