Improve docstrings for services and processors (#2087)

This commit is contained in:
Mark Backman
2025-06-28 13:39:45 -04:00
committed by GitHub
parent e1b0db75eb
commit 0ecfa827e6
117 changed files with 5136 additions and 862 deletions

View File

@@ -43,8 +43,8 @@ We follow Google-style docstrings with these specific conventions:
**Regular Classes:** **Regular Classes:**
- Class docstring describes the class purpose and documents all `__init__` parameters in an `Args:` section - Class docstring describes the class purpose and key functionality
- No separate `__init__` docstring needed - `__init__` method has its own docstring with complete `Args:` section documenting all parameters
- All public methods must have docstrings with `Args:` and `Returns:` sections as appropriate - All public methods must have docstrings with `Args:` and `Returns:` sections as appropriate
**Dataclasses:** **Dataclasses:**
@@ -60,6 +60,17 @@ We follow Google-style docstrings with these specific conventions:
- Must have docstrings explaining what subclasses should implement - Must have docstrings explaining what subclasses should implement
**`__init__.py` Files:**
- **Skip docstrings** for pure import/re-export modules
- **Add brief docstrings** for top-level packages or those with initialization logic
**Enums:**
- Class docstring describes the enumeration purpose
- Use `Parameters:` section to document each enum value and its meaning
- No `__init__` docstring (Enums don't have custom constructors)
#### Examples: #### Examples:
```python ```python
@@ -67,14 +78,18 @@ We follow Google-style docstrings with these specific conventions:
class MyService(BaseService): class MyService(BaseService):
"""Description of what the service does. """Description of what the service does.
Args: Provides detailed explanation of the service's functionality,
param1: Description of param1. key features, and usage patterns.
param2: Description of param2. Defaults to True.
**kwargs: Additional arguments passed to parent.
""" """
def __init__(self, param1: str, param2: bool = True, **kwargs): def __init__(self, param1: str, param2: bool = True, **kwargs):
# No docstring - parameters documented above """Initialize the service.
Args:
param1: Description of param1.
param2: Description of param2. Defaults to True.
**kwargs: Additional arguments passed to parent.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
@property @property
@@ -111,6 +126,22 @@ class ConfigParams:
host: str host: str
port: int = 8080 port: int = 8080
timeout: float = 30.0 timeout: float = 30.0
# Enum class
class Status(Enum):
"""Status codes for processing operations.
Parameters:
PENDING: Operation is queued but not started.
RUNNING: Operation is currently in progress.
COMPLETED: Operation finished successfully.
FAILED: Operation encountered an error.
"""
PENDING = "pending"
RUNNING = "running"
COMPLETED = "completed"
FAILED = "failed"
``` ```
# Contributor Covenant Code of Conduct # Contributor Covenant Code of Conduct

View File

@@ -28,15 +28,14 @@ extensions = [
# Napoleon settings # Napoleon settings
napoleon_google_docstring = True napoleon_google_docstring = True
napoleon_numpy_docstring = False napoleon_include_init_with_doc = True
napoleon_include_init_with_doc = False
# AutoDoc settings # AutoDoc settings
autodoc_default_options = { autodoc_default_options = {
"members": True, "members": True,
"member-order": "bysource", "member-order": "bysource",
"undoc-members": True, "undoc-members": True,
"exclude-members": "__weakref__,__init__", "exclude-members": "__weakref__,model_config",
"no-index": True, "no-index": True,
"show-inheritance": True, "show-inheritance": True,
} }
@@ -173,7 +172,7 @@ autodoc_mock_imports = [
# HTML output settings # HTML output settings
html_theme = "sphinx_rtd_theme" html_theme = "sphinx_rtd_theme"
html_static_path = ["_static"] html_static_path = ["_static"]
autodoc_typehints = "description" autodoc_typehints = "signature" # Show type hints in the signature only, not in the docstring
html_show_sphinx = False html_show_sphinx = False
@@ -275,6 +274,7 @@ def clean_title(title: str) -> str:
"stt": "STT", "stt": "STT",
"tts": "TTS", "tts": "TTS",
"llm": "LLM", "llm": "LLM",
"rtvi": "RTVI",
} }
# Check if the entire title is a special case # Check if the entire title is a special case

View File

@@ -123,8 +123,9 @@ select = [
"D", # Docstring rules "D", # Docstring rules
"I", # Import rules "I", # Import rules
] ]
# Ignore requirement for __init__ docstrings
ignore = ["D107"] [tool.ruff.lint.per-file-ignores]
"**/__init__.py" = ["D104"]
[tool.ruff.lint.pydocstyle] [tool.ruff.lint.pydocstyle]
convention = "google" convention = "google"

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""DTMF aggregation processor for converting keypad input to transcription.
This module provides a frame processor that aggregates DTMF (Dual-Tone Multi-Frequency)
keypad inputs into meaningful sequences and converts them to transcription frames
for downstream processing by LLM context aggregators.
"""
import asyncio import asyncio
from typing import Optional from typing import Optional
@@ -31,11 +38,6 @@ class DTMFAggregator(FrameProcessor):
- EndFrame or CancelFrame is received - EndFrame or CancelFrame is received
Emits TranscriptionFrame for compatibility with existing LLM context aggregators. Emits TranscriptionFrame for compatibility with existing LLM context aggregators.
Args:
timeout: Idle timeout in seconds before flushing
termination_digit: Digit that triggers immediate flush
prefix: Prefix added to DTMF sequence in transcription
""" """
def __init__( def __init__(
@@ -45,6 +47,14 @@ class DTMFAggregator(FrameProcessor):
prefix: str = "DTMF: ", prefix: str = "DTMF: ",
**kwargs, **kwargs,
): ):
"""Initialize the DTMF aggregator.
Args:
timeout: Idle timeout in seconds before flushing
termination_digit: Digit that triggers immediate flush
prefix: Prefix added to DTMF sequence in transcription
**kwargs: Additional arguments passed to FrameProcessor
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._aggregation = "" self._aggregation = ""
self._idle_timeout = timeout self._idle_timeout = timeout
@@ -55,6 +65,12 @@ class DTMFAggregator(FrameProcessor):
self._aggregation_task: Optional[asyncio.Task] = None self._aggregation_task: Optional[asyncio.Task] = None
async def process_frame(self, frame: Frame, direction: FrameDirection) -> None: async def process_frame(self, frame: Frame, direction: FrameDirection) -> None:
"""Process incoming frames and handle DTMF aggregation.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Gated frame aggregator for conditional frame accumulation.
This module provides a gated aggregator that accumulates frames based on
custom gate open/close functions, allowing for conditional frame buffering
and release in frame processing pipelines.
"""
from typing import List, Tuple from typing import List, Tuple
from loguru import logger from loguru import logger
@@ -14,8 +21,11 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class GatedAggregator(FrameProcessor): class GatedAggregator(FrameProcessor):
"""Accumulate frames, with custom functions to start and stop accumulation. """Accumulate frames, with custom functions to start and stop accumulation.
Yields gate-opening frame before any accumulated frames, then ensuing frames Yields gate-opening frame before any accumulated frames, then ensuing frames
until and not including the gate-closed frame. until and not including the gate-closed frame. The aggregator maintains an
internal gate state that controls whether frames are passed through immediately
or accumulated for later release.
Doctest: FIXME to work with asyncio Doctest: FIXME to work with asyncio
>>> from pipecat.frames.frames import ImageRawFrame >>> from pipecat.frames.frames import ImageRawFrame
@@ -48,6 +58,14 @@ class GatedAggregator(FrameProcessor):
start_open, start_open,
direction: FrameDirection = FrameDirection.DOWNSTREAM, direction: FrameDirection = FrameDirection.DOWNSTREAM,
): ):
"""Initialize the gated aggregator.
Args:
gate_open_fn: Function that returns True when a frame should open the gate.
gate_close_fn: Function that returns True when a frame should close the gate.
start_open: Whether the gate should start in the open state.
direction: The frame direction this aggregator operates on.
"""
super().__init__() super().__init__()
self._gate_open_fn = gate_open_fn self._gate_open_fn = gate_open_fn
self._gate_close_fn = gate_close_fn self._gate_close_fn = gate_close_fn
@@ -56,6 +74,12 @@ class GatedAggregator(FrameProcessor):
self._accumulator: List[Tuple[Frame, FrameDirection]] = [] self._accumulator: List[Tuple[Frame, FrameDirection]] = []
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames with gated accumulation logic.
Args:
frame: The frame to process.
direction: The direction of the frame flow.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# We must not block system frames. # We must not block system frames.

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Gated OpenAI LLM context aggregator for controlled message flow."""
from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame
from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -11,12 +13,21 @@ from pipecat.sync.base_notifier import BaseNotifier
class GatedOpenAILLMContextAggregator(FrameProcessor): class GatedOpenAILLMContextAggregator(FrameProcessor):
"""This aggregator keeps the last received OpenAI LLM context frame and it """Aggregator that gates OpenAI LLM context frames until notified.
doesn't let it through until the notifier is notified.
This aggregator captures OpenAI LLM context frames and holds them until
a notifier signals that they can be released. This is useful for controlling
the flow of context frames based on external conditions or timing.
""" """
def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs): def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs):
"""Initialize the gated context aggregator.
Args:
notifier: The notifier that controls when frames are released.
start_open: If True, the first context frame passes through immediately.
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._notifier = notifier self._notifier = notifier
self._start_open = start_open self._start_open = start_open
@@ -24,6 +35,12 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
self._gate_task = None self._gate_task = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames, gating OpenAI LLM context frames.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
@@ -42,15 +59,18 @@ class GatedOpenAILLMContextAggregator(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def _start(self): async def _start(self):
"""Start the gate task handler."""
if not self._gate_task: if not self._gate_task:
self._gate_task = self.create_task(self._gate_task_handler()) self._gate_task = self.create_task(self._gate_task_handler())
async def _stop(self): async def _stop(self):
"""Stop the gate task handler."""
if self._gate_task: if self._gate_task:
await self.cancel_task(self._gate_task) await self.cancel_task(self._gate_task)
self._gate_task = None self._gate_task = None
async def _gate_task_handler(self): async def _gate_task_handler(self):
"""Handle the gating logic by waiting for notifications and releasing frames."""
while True: while True:
await self._notifier.wait() await self._notifier.wait()
if self._last_context_frame: if self._last_context_frame:

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""LLM response aggregators for handling conversation context and message aggregation.
This module provides aggregators that process and accumulate LLM responses, user inputs,
and conversation context. These aggregators handle the flow between speech-to-text,
LLM processing, and text-to-speech components in conversational AI pipelines.
"""
import asyncio import asyncio
from abc import abstractmethod from abc import abstractmethod
from dataclasses import dataclass from dataclasses import dataclass
@@ -54,30 +61,55 @@ from pipecat.utils.time import time_now_iso8601
@dataclass @dataclass
class LLMUserAggregatorParams: class LLMUserAggregatorParams:
"""Parameters for configuring LLM user aggregation behavior.
Parameters:
aggregation_timeout: Maximum time in seconds to wait for additional
transcription content before pushing aggregated result. This
timeout is used only when the transcription is slow to arrive.
"""
aggregation_timeout: float = 0.5 aggregation_timeout: float = 0.5
@dataclass @dataclass
class LLMAssistantAggregatorParams: class LLMAssistantAggregatorParams:
"""Parameters for configuring LLM assistant aggregation behavior.
Parameters:
expect_stripped_words: Whether to expect and handle stripped words
in text frames by adding spaces between tokens.
"""
expect_stripped_words: bool = True expect_stripped_words: bool = True
class LLMFullResponseAggregator(FrameProcessor): class LLMFullResponseAggregator(FrameProcessor):
"""This is an LLM aggregator that aggregates a full LLM completion. It """Aggregates complete LLM responses between start and end frames.
aggregates LLM text frames (tokens) received between
`LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. Every full
completion is returned via the "on_completion" event handler:
@aggregator.event_handler("on_completion") This aggregator collects LLM text frames (tokens) received between
async def on_completion( `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame` and provides
aggregator: LLMFullResponseAggregator, the complete response via an event handler.
completion: str,
completed: bool,
)
The aggregator provides an "on_completion" event that fires when a full
completion is available:
@aggregator.event_handler("on_completion")
async def on_completion(
aggregator: LLMFullResponseAggregator,
completion: str,
completed: bool,
):
# Handle the completion
pass
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the LLM full response aggregator.
Args:
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._aggregation = "" self._aggregation = ""
@@ -86,6 +118,12 @@ class LLMFullResponseAggregator(FrameProcessor):
self._register_event_handler("on_completion") self._register_event_handler("on_completion")
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and aggregate LLM text content.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame): if isinstance(frame, StartInterruptionFrame):
@@ -116,83 +154,123 @@ class LLMFullResponseAggregator(FrameProcessor):
class BaseLLMResponseAggregator(FrameProcessor): class BaseLLMResponseAggregator(FrameProcessor):
"""This is the base class for all LLM response aggregators. These """Base class for all LLM response aggregators.
aggregators process incoming frames and aggregate content until they are
ready to push the aggregation. In the case of a user, an aggregation might
be a full transcription received from the STT service.
The LLM response aggregators also keep a store (e.g. a message list or an These aggregators process incoming frames and aggregate content until they are
LLM context) of the current conversation, that is, it stores the messages ready to push the aggregation downstream. They maintain conversation state
said by the user or by the bot. and handle message flow between different components in the pipeline.
The aggregators keep a store (e.g. message list or LLM context) of the current
conversation, storing messages from both users and the bot.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the base LLM response aggregator.
Args:
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
@property @property
@abstractmethod @abstractmethod
def messages(self) -> List[dict]: def messages(self) -> List[dict]:
"""Returns the messages from the current conversation.""" """Get the messages from the current conversation.
Returns:
List of message dictionaries representing the conversation history.
"""
pass pass
@property @property
@abstractmethod @abstractmethod
def role(self) -> str: def role(self) -> str:
"""Returns the role (e.g. user, assistant...) for this aggregator.""" """Get the role for this aggregator.
Returns:
The role string (e.g. "user", "assistant") for this aggregator.
"""
pass pass
@abstractmethod @abstractmethod
def add_messages(self, messages): def add_messages(self, messages):
"""Add the given messages to the conversation.""" """Add the given messages to the conversation.
Args:
messages: Messages to append to the conversation history.
"""
pass pass
@abstractmethod @abstractmethod
def set_messages(self, messages): def set_messages(self, messages):
"""Reset the conversation with the given messages.""" """Reset the conversation with the given messages.
Args:
messages: Messages to replace the current conversation history.
"""
pass pass
@abstractmethod @abstractmethod
def set_tools(self, tools): def set_tools(self, tools):
"""Set LLM tools to be used in the current conversation.""" """Set LLM tools to be used in the current conversation.
Args:
tools: List of tool definitions for the LLM to use.
"""
pass pass
@abstractmethod @abstractmethod
def set_tool_choice(self, tool_choice): def set_tool_choice(self, tool_choice):
"""Set the tool choice. This should modify the LLM context.""" """Set the tool choice for the LLM.
Args:
tool_choice: Tool choice configuration for the LLM context.
"""
pass pass
@abstractmethod @abstractmethod
async def reset(self): async def reset(self):
"""Reset the internals of this aggregator. This should not modify the """Reset the internal state of this aggregator.
internal messages.
This should clear aggregation state but not modify the conversation messages.
""" """
pass pass
@abstractmethod @abstractmethod
async def handle_aggregation(self, aggregation: str): async def handle_aggregation(self, aggregation: str):
"""Adds the given aggregation to the aggregator. The aggregator can use """Add the given aggregation to the conversation store.
a simple list of message or a context. It doesn't not push any frames.
Args:
aggregation: The aggregated text content to add to the conversation.
""" """
pass pass
@abstractmethod @abstractmethod
async def push_aggregation(self): async def push_aggregation(self):
"""Pushes the current aggregation. For example, iN the case of context """Push the current aggregation downstream.
aggregation this might push a new context frame.
The specific frame type pushed depends on the aggregator implementation
(e.g. context frame, messages frame).
""" """
pass pass
class LLMContextResponseAggregator(BaseLLMResponseAggregator): class LLMContextResponseAggregator(BaseLLMResponseAggregator):
"""This is a base LLM aggregator that uses an LLM context to store the """Base LLM aggregator that uses an OpenAI LLM context for conversation storage.
conversation. It pushes `OpenAILLMContextFrame` as an aggregation frame.
This aggregator maintains conversation state using an OpenAILLMContext and
pushes OpenAILLMContextFrame objects as aggregation frames. It provides
common functionality for context-based conversation management.
""" """
def __init__(self, *, context: OpenAILLMContext, role: str, **kwargs): def __init__(self, *, context: OpenAILLMContext, role: str, **kwargs):
"""Initialize the context response aggregator.
Args:
context: The OpenAI LLM context to use for conversation storage.
role: The role this aggregator represents (e.g. "user", "assistant").
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._context = context self._context = context
self._role = role self._role = role
@@ -201,46 +279,98 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator):
@property @property
def messages(self) -> List[dict]: def messages(self) -> List[dict]:
"""Get messages from the LLM context.
Returns:
List of message dictionaries from the context.
"""
return self._context.get_messages() return self._context.get_messages()
@property @property
def role(self) -> str: def role(self) -> str:
"""Get the role for this aggregator.
Returns:
The role string for this aggregator.
"""
return self._role return self._role
@property @property
def context(self): def context(self):
"""Get the OpenAI LLM context.
Returns:
The OpenAILLMContext instance used by this aggregator.
"""
return self._context return self._context
def get_context_frame(self) -> OpenAILLMContextFrame: def get_context_frame(self) -> OpenAILLMContextFrame:
"""Create a context frame with the current context.
Returns:
OpenAILLMContextFrame containing the current context.
"""
return OpenAILLMContextFrame(context=self._context) return OpenAILLMContextFrame(context=self._context)
async def push_context_frame(self, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_context_frame(self, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a context frame in the specified direction.
Args:
direction: The direction to push the frame (upstream or downstream).
"""
frame = self.get_context_frame() frame = self.get_context_frame()
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
def add_messages(self, messages): def add_messages(self, messages):
"""Add messages to the context.
Args:
messages: Messages to add to the conversation context.
"""
self._context.add_messages(messages) self._context.add_messages(messages)
def set_messages(self, messages): def set_messages(self, messages):
"""Set the context messages.
Args:
messages: Messages to replace the current context messages.
"""
self._context.set_messages(messages) self._context.set_messages(messages)
def set_tools(self, tools: List): def set_tools(self, tools: List):
"""Set tools in the context.
Args:
tools: List of tool definitions to set in the context.
"""
self._context.set_tools(tools) self._context.set_tools(tools)
def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict): def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict):
"""Set tool choice in the context.
Args:
tool_choice: Tool choice configuration for the context.
"""
self._context.set_tool_choice(tool_choice) self._context.set_tool_choice(tool_choice)
async def reset(self): async def reset(self):
"""Reset the aggregation state."""
self._aggregation = "" self._aggregation = ""
class LLMUserContextAggregator(LLMContextResponseAggregator): class LLMUserContextAggregator(LLMContextResponseAggregator):
"""This is a user LLM aggregator that uses an LLM context to store the """User LLM aggregator that processes speech-to-text transcriptions.
conversation. It aggregates transcriptions from the STT service and it has
logic to handle multiple scenarios where transcriptions are received between
VAD events (`UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame`) or
even outside or no VAD events at all.
This aggregator handles the complex logic of aggregating user speech transcriptions
from STT services. It manages multiple scenarios including:
- Transcriptions received between VAD events
- Transcriptions received outside VAD events
- Interim vs final transcriptions
- User interruptions during bot speech
- Emulated VAD for whispered or short utterances
The aggregator uses timeouts to handle cases where transcriptions arrive
after VAD events or when no VAD is available.
""" """
def __init__( def __init__(
@@ -250,6 +380,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
params: Optional[LLMUserAggregatorParams] = None, params: Optional[LLMUserAggregatorParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the user context aggregator.
Args:
context: The OpenAI LLM context for conversation storage.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments. Supports deprecated 'aggregation_timeout'.
"""
super().__init__(context=context, role="user", **kwargs) super().__init__(context=context, role="user", **kwargs)
self._params = params or LLMUserAggregatorParams() self._params = params or LLMUserAggregatorParams()
if "aggregation_timeout" in kwargs: if "aggregation_timeout" in kwargs:
@@ -275,6 +412,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
self._aggregation_task = None self._aggregation_task = None
async def reset(self): async def reset(self):
"""Reset the aggregation state and interruption strategies."""
await super().reset() await super().reset()
self._was_bot_speaking = False self._was_bot_speaking = False
self._seen_interim_results = False self._seen_interim_results = False
@@ -282,9 +420,20 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
[await s.reset() for s in self._interruption_strategies] [await s.reset() for s in self._interruption_strategies]
async def handle_aggregation(self, aggregation: str): async def handle_aggregation(self, aggregation: str):
"""Add the aggregated user text to the context.
Args:
aggregation: The aggregated user text to add as a user message.
"""
self._context.add_message({"role": self.role, "content": aggregation}) self._context.add_message({"role": self.role, "content": aggregation})
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for user speech aggregation and context management.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
@@ -339,7 +488,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
await self.push_frame(frame) await self.push_frame(frame)
async def push_aggregation(self): async def push_aggregation(self):
"""Pushes the current aggregation based on interruption strategies and conditions.""" """Push the current aggregation based on interruption strategies and conditions."""
if len(self._aggregation) > 0: if len(self._aggregation) > 0:
if self.interruption_strategies and self._bot_speaking: if self.interruption_strategies and self._bot_speaking:
should_interrupt = await self._should_interrupt_based_on_strategies() should_interrupt = await self._should_interrupt_based_on_strategies()
@@ -373,7 +522,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
# await self.push_frame(OpenAILLMContextFrame(self._context)) # await self.push_frame(OpenAILLMContextFrame(self._context))
async def _should_interrupt_based_on_strategies(self) -> bool: async def _should_interrupt_based_on_strategies(self) -> bool:
"""Check if interruption should occur based on configured strategies.""" """Check if interruption should occur based on configured strategies.
Returns:
True if any interruption strategy indicates interruption should occur.
"""
async def should_interrupt(strategy: BaseInterruptionStrategy): async def should_interrupt(strategy: BaseInterruptionStrategy):
await strategy.append_text(self._aggregation) await strategy.append_text(self._aggregation)
@@ -474,9 +627,10 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
self._aggregation_event.clear() self._aggregation_event.clear()
async def _maybe_emulate_user_speaking(self): async def _maybe_emulate_user_speaking(self):
"""Emulate user speaking if we got a transcription but it was not """Maybe emulate user speaking based on transcription.
detected by VAD. Only do that if the bot is not speaking.
Emulate user speaking if we got a transcription but it was not
detected by VAD. Only do that if the bot is not speaking.
""" """
# Check if we received a transcription but VAD was not able to detect # Check if we received a transcription but VAD was not able to detect
# voice (e.g. when you whisper a short utterance). In that case, we need # voice (e.g. when you whisper a short utterance). In that case, we need
@@ -497,10 +651,17 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
class LLMAssistantContextAggregator(LLMContextResponseAggregator): class LLMAssistantContextAggregator(LLMContextResponseAggregator):
"""This is an assistant LLM aggregator that uses an LLM context to store the """Assistant LLM aggregator that processes bot responses and function calls.
conversation. It aggregates text frames received between
`LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`.
This aggregator handles the complex logic of processing assistant responses including:
- Text frame aggregation between response start/end markers
- Function call lifecycle management
- Context updates with timestamps
- Tool execution and result handling
- Interruption handling during responses
The aggregator manages function calls in progress and coordinates between
text generation and tool execution phases of LLM responses.
""" """
def __init__( def __init__(
@@ -510,6 +671,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
params: Optional[LLMAssistantAggregatorParams] = None, params: Optional[LLMAssistantAggregatorParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the assistant context aggregator.
Args:
context: The OpenAI LLM context for conversation storage.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments. Supports deprecated 'expect_stripped_words'.
"""
super().__init__(context=context, role="assistant", **kwargs) super().__init__(context=context, role="assistant", **kwargs)
self._params = params or LLMAssistantAggregatorParams() self._params = params or LLMAssistantAggregatorParams()
@@ -534,26 +702,57 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
"""Check if there are any function calls currently in progress. """Check if there are any function calls currently in progress.
Returns: Returns:
bool: True if function calls are in progress, False otherwise True if function calls are in progress, False otherwise.
""" """
return bool(self._function_calls_in_progress) return bool(self._function_calls_in_progress)
async def handle_aggregation(self, aggregation: str): async def handle_aggregation(self, aggregation: str):
"""Add the aggregated assistant text to the context.
Args:
aggregation: The aggregated assistant text to add as an assistant message.
"""
self._context.add_message({"role": "assistant", "content": aggregation}) self._context.add_message({"role": "assistant", "content": aggregation})
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
"""Handle a function call that is in progress.
Args:
frame: The function call in progress frame to handle.
"""
pass pass
async def handle_function_call_result(self, frame: FunctionCallResultFrame): async def handle_function_call_result(self, frame: FunctionCallResultFrame):
"""Handle the result of a completed function call.
Args:
frame: The function call result frame to handle.
"""
pass pass
async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
"""Handle cancellation of a function call.
Args:
frame: The function call cancel frame to handle.
"""
pass pass
async def handle_user_image_frame(self, frame: UserImageRawFrame): async def handle_user_image_frame(self, frame: UserImageRawFrame):
"""Handle a user image frame associated with a function call.
Args:
frame: The user image frame to handle.
"""
pass pass
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for assistant response aggregation and function call management.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame): if isinstance(frame, StartInterruptionFrame):
@@ -590,6 +789,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def push_aggregation(self): async def push_aggregation(self):
"""Push the current assistant aggregation with timestamp."""
if not self._aggregation: if not self._aggregation:
return return
@@ -719,6 +919,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
class LLMUserResponseAggregator(LLMUserContextAggregator): class LLMUserResponseAggregator(LLMUserContextAggregator):
"""User response aggregator that outputs LLMMessagesFrame instead of context frames.
This aggregator extends LLMUserContextAggregator but pushes LLMMessagesFrame
objects downstream instead of OpenAILLMContextFrame objects. This is useful
when you need message-based output rather than context-based output.
"""
def __init__( def __init__(
self, self,
messages: Optional[List[dict]] = None, messages: Optional[List[dict]] = None,
@@ -726,9 +933,17 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
params: Optional[LLMUserAggregatorParams] = None, params: Optional[LLMUserAggregatorParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the user response aggregator.
Args:
messages: Initial messages for the conversation context.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs) super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)
async def push_aggregation(self): async def push_aggregation(self):
"""Push the aggregated user input as an LLMMessagesFrame."""
if len(self._aggregation) > 0: if len(self._aggregation) > 0:
await self.handle_aggregation(self._aggregation) await self.handle_aggregation(self._aggregation)
@@ -741,6 +956,13 @@ class LLMUserResponseAggregator(LLMUserContextAggregator):
class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
"""Assistant response aggregator that outputs LLMMessagesFrame instead of context frames.
This aggregator extends LLMAssistantContextAggregator but pushes LLMMessagesFrame
objects downstream instead of OpenAILLMContextFrame objects. This is useful
when you need message-based output rather than context-based output.
"""
def __init__( def __init__(
self, self,
messages: Optional[List[dict]] = None, messages: Optional[List[dict]] = None,
@@ -748,9 +970,17 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator):
params: Optional[LLMAssistantAggregatorParams] = None, params: Optional[LLMAssistantAggregatorParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the assistant response aggregator.
Args:
messages: Initial messages for the conversation context.
params: Configuration parameters for aggregation behavior.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs) super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs)
async def push_aggregation(self): async def push_aggregation(self):
"""Push the aggregated assistant response as an LLMMessagesFrame."""
if len(self._aggregation) > 0: if len(self._aggregation) > 0:
await self.handle_aggregation(self._aggregation) await self.handle_aggregation(self._aggregation)

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""OpenAI LLM context management for Pipecat.
This module provides classes for managing OpenAI-specific conversation contexts,
including message handling, tool management, and image/audio processing capabilities.
"""
import base64 import base64
import copy import copy
import io import io
@@ -29,7 +35,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class CustomEncoder(json.JSONEncoder): class CustomEncoder(json.JSONEncoder):
"""Custom JSON encoder for handling special data types in logging.
Provides specialized encoding for io.BytesIO objects to display
readable representations in log output instead of raw binary data.
"""
def default(self, obj): def default(self, obj):
"""Encode special objects for JSON serialization.
Args:
obj: The object to encode.
Returns:
Encoded representation of the object.
"""
if isinstance(obj, io.BytesIO): if isinstance(obj, io.BytesIO):
# Convert the first 8 bytes to an ASCII hex string # Convert the first 8 bytes to an ASCII hex string
return f"{obj.getbuffer()[0:8].hex()}..." return f"{obj.getbuffer()[0:8].hex()}..."
@@ -37,25 +57,57 @@ class CustomEncoder(json.JSONEncoder):
class OpenAILLMContext: class OpenAILLMContext:
"""Manages conversation context for OpenAI LLM interactions.
Handles message history, tool definitions, tool choices, and multimedia content
for OpenAI API conversations. Provides methods for message manipulation,
content formatting, and integration with various LLM adapters.
"""
def __init__( def __init__(
self, self,
messages: Optional[List[ChatCompletionMessageParam]] = None, messages: Optional[List[ChatCompletionMessageParam]] = None,
tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN,
tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN,
): ):
"""Initialize the OpenAI LLM context.
Args:
messages: Initial list of conversation messages.
tools: Available tools for the LLM to use.
tool_choice: Tool selection strategy for the LLM.
"""
self._messages: List[ChatCompletionMessageParam] = messages if messages else [] self._messages: List[ChatCompletionMessageParam] = messages if messages else []
self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice
self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools
self._llm_adapter: Optional[BaseLLMAdapter] = None self._llm_adapter: Optional[BaseLLMAdapter] = None
def get_llm_adapter(self) -> Optional[BaseLLMAdapter]: def get_llm_adapter(self) -> Optional[BaseLLMAdapter]:
"""Get the current LLM adapter.
Returns:
The currently set LLM adapter, or None if not set.
"""
return self._llm_adapter return self._llm_adapter
def set_llm_adapter(self, llm_adapter: BaseLLMAdapter): def set_llm_adapter(self, llm_adapter: BaseLLMAdapter):
"""Set the LLM adapter for context processing.
Args:
llm_adapter: The LLM adapter to use for tool conversion.
"""
self._llm_adapter = llm_adapter self._llm_adapter = llm_adapter
@staticmethod @staticmethod
def from_messages(messages: List[dict]) -> "OpenAILLMContext": def from_messages(messages: List[dict]) -> "OpenAILLMContext":
"""Create a context from a list of message dictionaries.
Args:
messages: List of message dictionaries to convert to context.
Returns:
New OpenAILLMContext instance with the provided messages.
"""
context = OpenAILLMContext() context = OpenAILLMContext()
for message in messages: for message in messages:
@@ -66,34 +118,81 @@ class OpenAILLMContext:
@property @property
def messages(self) -> List[ChatCompletionMessageParam]: def messages(self) -> List[ChatCompletionMessageParam]:
"""Get the current messages list.
Returns:
List of conversation messages.
"""
return self._messages return self._messages
@property @property
def tools(self) -> List[ChatCompletionToolParam] | NotGiven | List[Any]: def tools(self) -> List[ChatCompletionToolParam] | NotGiven | List[Any]:
"""Get the tools list, converting through adapter if available.
Returns:
Tools list, potentially converted by the LLM adapter.
"""
if self._llm_adapter: if self._llm_adapter:
return self._llm_adapter.from_standard_tools(self._tools) return self._llm_adapter.from_standard_tools(self._tools)
return self._tools return self._tools
@property @property
def tool_choice(self) -> ChatCompletionToolChoiceOptionParam | NotGiven: def tool_choice(self) -> ChatCompletionToolChoiceOptionParam | NotGiven:
"""Get the current tool choice setting.
Returns:
The tool choice configuration.
"""
return self._tool_choice return self._tool_choice
def add_message(self, message: ChatCompletionMessageParam): def add_message(self, message: ChatCompletionMessageParam):
"""Add a single message to the context.
Args:
message: The message to add to the conversation history.
"""
self._messages.append(message) self._messages.append(message)
def add_messages(self, messages: List[ChatCompletionMessageParam]): def add_messages(self, messages: List[ChatCompletionMessageParam]):
"""Add multiple messages to the context.
Args:
messages: List of messages to add to the conversation history.
"""
self._messages.extend(messages) self._messages.extend(messages)
def set_messages(self, messages: List[ChatCompletionMessageParam]): def set_messages(self, messages: List[ChatCompletionMessageParam]):
"""Replace all messages in the context.
Args:
messages: New list of messages to replace the current history.
"""
self._messages[:] = messages self._messages[:] = messages
def get_messages(self) -> List[ChatCompletionMessageParam]: def get_messages(self) -> List[ChatCompletionMessageParam]:
"""Get a copy of the current messages list.
Returns:
List of all messages in the conversation history.
"""
return self._messages return self._messages
def get_messages_json(self) -> str: def get_messages_json(self) -> str:
"""Get messages as a formatted JSON string.
Returns:
JSON string representation of all messages with custom encoding.
"""
return json.dumps(self._messages, cls=CustomEncoder, ensure_ascii=False, indent=2) return json.dumps(self._messages, cls=CustomEncoder, ensure_ascii=False, indent=2)
def get_messages_for_logging(self) -> str: def get_messages_for_logging(self) -> str:
"""Get sanitized messages suitable for logging.
Removes or truncates sensitive data like image content for safe logging.
Returns:
JSON string with sanitized message content for logging.
"""
msgs = [] msgs = []
for message in self.messages: for message in self.messages:
msg = copy.deepcopy(message) msg = copy.deepcopy(message)
@@ -118,10 +217,10 @@ class OpenAILLMContext:
Since OpenAI is our standard format, this is a passthrough function. Since OpenAI is our standard format, this is a passthrough function.
Args: Args:
message (dict): Message in OpenAI format message: Message in OpenAI format.
Returns: Returns:
dict: Same message, unchanged Same message, unchanged.
""" """
return message return message
@@ -133,20 +232,28 @@ class OpenAILLMContext:
other LLM services that may need to return multiple messages. other LLM services that may need to return multiple messages.
Args: Args:
obj (dict): Message in OpenAI format with either: obj: Message in OpenAI format with either simple string content
- Simple content: {"role": "user", "content": "Hello"} or structured list content.
- List content: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
Returns: Returns:
list: List containing the original messages, preserving whether List containing the original messages, preserving the content format.
the content was in simple string or structured list format
""" """
return [obj] return [obj]
def get_messages_for_initializing_history(self): def get_messages_for_initializing_history(self):
"""Get messages for initializing conversation history.
Returns:
List of messages suitable for history initialization.
"""
return self._messages return self._messages
def get_messages_for_persistent_storage(self): def get_messages_for_persistent_storage(self):
"""Get messages formatted for persistent storage.
Returns:
List of messages converted to standard format for storage.
"""
messages = [] messages = []
for m in self._messages: for m in self._messages:
standard_messages = self.to_standard_messages(m) standard_messages = self.to_standard_messages(m)
@@ -154,9 +261,19 @@ class OpenAILLMContext:
return messages return messages
def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven): def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven):
"""Set the tool choice configuration.
Args:
tool_choice: Tool selection strategy for the LLM.
"""
self._tool_choice = tool_choice self._tool_choice = tool_choice
def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN): def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN):
"""Set the available tools for the LLM.
Args:
tools: List of tools available to the LLM, or NOT_GIVEN to disable tools.
"""
if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0: if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0:
tools = NOT_GIVEN tools = NOT_GIVEN
self._tools = tools self._tools = tools
@@ -164,6 +281,14 @@ class OpenAILLMContext:
def add_image_frame_message( def add_image_frame_message(
self, *, format: str, size: tuple[int, int], image: bytes, text: str = None self, *, format: str, size: tuple[int, int], image: bytes, text: str = None
): ):
"""Add a message containing an image frame.
Args:
format: Image format (e.g., 'RGB', 'RGBA').
size: Image dimensions as (width, height) tuple.
image: Raw image bytes.
text: Optional text to include with the image.
"""
buffer = io.BytesIO() buffer = io.BytesIO()
Image.frombytes(format, size, image).save(buffer, format="JPEG") Image.frombytes(format, size, image).save(buffer, format="JPEG")
encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8")
@@ -177,10 +302,30 @@ class OpenAILLMContext:
self.add_message({"role": "user", "content": content}) self.add_message({"role": "user", "content": content})
def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None): def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None):
"""Add a message containing audio frames.
Args:
audio_frames: List of audio frame objects to include.
text: Optional text to include with the audio.
Note:
This method is currently a placeholder for future implementation.
"""
# todo: implement for OpenAI models and others # todo: implement for OpenAI models and others
pass pass
def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size): def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size):
"""Create a WAV file header for audio data.
Args:
sample_rate: Audio sample rate in Hz.
num_channels: Number of audio channels.
bits_per_sample: Bits per audio sample.
data_size: Size of audio data in bytes.
Returns:
WAV header as a bytearray.
"""
# RIFF chunk descriptor # RIFF chunk descriptor
header = bytearray() header = bytearray()
header.extend(b"RIFF") # ChunkID header.extend(b"RIFF") # ChunkID
@@ -206,10 +351,14 @@ class OpenAILLMContext:
@dataclass @dataclass
class OpenAILLMContextFrame(Frame): class OpenAILLMContextFrame(Frame):
"""Like an LLMMessagesFrame, but with extra context specific to the OpenAI """Frame containing OpenAI-specific LLM context.
Like an LLMMessagesFrame, but with extra context specific to the OpenAI
API. The context in this message is also mutable, and will be changed by the API. The context in this message is also mutable, and will be changed by the
OpenAIContextAggregator frame processor. OpenAIContextAggregator frame processor.
Parameters:
context: The OpenAI LLM context containing messages, tools, and configuration.
""" """
context: OpenAILLMContext context: OpenAILLMContext

View File

@@ -4,17 +4,28 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Text sentence aggregation processor for Pipecat.
This module provides a frame processor that accumulates text frames into
complete sentences, only outputting when a sentence-ending pattern is detected.
"""
from pipecat.frames.frames import EndFrame, Frame, InterimTranscriptionFrame, TextFrame from pipecat.frames.frames import EndFrame, Frame, InterimTranscriptionFrame, TextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.utils.string import match_endofsentence from pipecat.utils.string import match_endofsentence
class SentenceAggregator(FrameProcessor): class SentenceAggregator(FrameProcessor):
"""This frame processor aggregates text frames into complete sentences. """Aggregates text frames into complete sentences.
This processor accumulates incoming text frames until a sentence-ending
pattern is detected, then outputs the complete sentence as a single frame.
Useful for ensuring downstream processors receive coherent, complete sentences
rather than fragmented text.
Frame input/output: Frame input/output:
TextFrame("Hello,") -> None TextFrame("Hello,") -> None
TextFrame(" world.") -> TextFrame("Hello world.") TextFrame(" world.") -> TextFrame("Hello, world.")
Doctest: FIXME to work with asyncio Doctest: FIXME to work with asyncio
>>> import asyncio >>> import asyncio
@@ -29,10 +40,20 @@ class SentenceAggregator(FrameProcessor):
""" """
def __init__(self): def __init__(self):
"""Initialize the sentence aggregator.
Sets up internal state for accumulating text frames into complete sentences.
"""
super().__init__() super().__init__()
self._aggregation = "" self._aggregation = ""
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and aggregate text into complete sentences.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# We ignore interim description at this point. # We ignore interim description at this point.

View File

@@ -4,15 +4,39 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""User response aggregation for text frames.
This module provides an aggregator that collects user responses and outputs
them as TextFrame objects, useful for capturing and processing user input
in conversational pipelines.
"""
from pipecat.frames.frames import TextFrame from pipecat.frames.frames import TextFrame
from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator
class UserResponseAggregator(LLMUserResponseAggregator): class UserResponseAggregator(LLMUserResponseAggregator):
"""Aggregates user responses into TextFrame objects.
This aggregator extends LLMUserResponseAggregator to specifically handle
user input by collecting text responses and outputting them as TextFrame
objects when the aggregation is complete.
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the user response aggregator.
Args:
**kwargs: Additional arguments passed to parent LLMUserResponseAggregator.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
async def push_aggregation(self): async def push_aggregation(self):
"""Push the aggregated user response as a TextFrame.
Creates a TextFrame from the current aggregation if it contains content,
resets the aggregation state, and pushes the frame downstream.
"""
if len(self._aggregation) > 0: if len(self._aggregation) > 0:
frame = TextFrame(self._aggregation.strip()) frame = TextFrame(self._aggregation.strip())

View File

@@ -4,14 +4,22 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Vision image frame aggregation for Pipecat.
This module provides frame aggregation functionality to combine text and image
frames into vision frames for multimodal processing.
"""
from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class VisionImageFrameAggregator(FrameProcessor): class VisionImageFrameAggregator(FrameProcessor):
"""This aggregator waits for a consecutive TextFrame and an """Aggregates consecutive text and image frames into vision frames.
InputImageRawFrame. After the InputImageRawFrame arrives it will output a
VisionImageRawFrame. This aggregator waits for a consecutive TextFrame and an InputImageRawFrame.
After the InputImageRawFrame arrives it will output a VisionImageRawFrame
combining both the text and image data for multimodal processing.
>>> from pipecat.frames.frames import ImageFrame >>> from pipecat.frames.frames import ImageFrame
@@ -23,14 +31,27 @@ class VisionImageFrameAggregator(FrameProcessor):
>>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?"))) >>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?")))
>>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0)))) >>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0))))
VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B
""" """
def __init__(self): def __init__(self):
"""Initialize the vision image frame aggregator.
The aggregator starts with no cached text, waiting for the first
TextFrame to arrive before it can create vision frames.
"""
super().__init__() super().__init__()
self._describe_text = None self._describe_text = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and aggregate text with images.
Caches TextFrames and combines them with subsequent InputImageRawFrames
to create VisionImageRawFrames. Other frames are passed through unchanged.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Async generator processor for frame serialization and streaming."""
import asyncio import asyncio
from typing import Any, AsyncGenerator from typing import Any, AsyncGenerator
@@ -17,12 +19,32 @@ from pipecat.serializers.base_serializer import FrameSerializer
class AsyncGeneratorProcessor(FrameProcessor): class AsyncGeneratorProcessor(FrameProcessor):
"""A frame processor that serializes frames and provides them via async generator.
This processor passes frames through unchanged while simultaneously serializing
them and making the serialized data available through an async generator interface.
Useful for streaming frame data to external consumers while maintaining the
normal frame processing pipeline.
"""
def __init__(self, *, serializer: FrameSerializer, **kwargs): def __init__(self, *, serializer: FrameSerializer, **kwargs):
"""Initialize the async generator processor.
Args:
serializer: The frame serializer to use for converting frames to data.
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._serializer = serializer self._serializer = serializer
self._data_queue = asyncio.Queue() self._data_queue = asyncio.Queue()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames by passing them through and queuing serialized data.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
@@ -35,6 +57,12 @@ class AsyncGeneratorProcessor(FrameProcessor):
await self._data_queue.put(data) await self._data_queue.put(data)
async def generator(self) -> AsyncGenerator[Any, None]: async def generator(self) -> AsyncGenerator[Any, None]:
"""Generate serialized frame data asynchronously.
Yields:
Serialized frame data from the internal queue until a termination
signal (None) is received.
"""
running = True running = True
while running: while running:
data = await self._data_queue.get() data = await self._data_queue.get()

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Audio buffer processor for managing and synchronizing audio streams.
This module provides an AudioBufferProcessor that handles buffering and synchronization
of audio from both user input and bot output sources, with support for various audio
configurations and event-driven processing.
"""
import time import time
from typing import Optional from typing import Optional
@@ -37,12 +44,6 @@ class AudioBufferProcessor(FrameProcessor):
on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio
on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio
Args:
sample_rate (Optional[int]): Desired output sample rate. If None, uses source rate
num_channels (int): Number of channels (1 for mono, 2 for stereo). Defaults to 1
buffer_size (int): Size of buffer before triggering events. 0 for no buffering
enable_turn_audio (bool): Whether turn audio event handlers should be triggered
Audio handling: Audio handling:
- Mono output (num_channels=1): User and bot audio are mixed - Mono output (num_channels=1): User and bot audio are mixed
- Stereo output (num_channels=2): User audio on left, bot audio on right - Stereo output (num_channels=2): User audio on left, bot audio on right
@@ -61,6 +62,16 @@ class AudioBufferProcessor(FrameProcessor):
enable_turn_audio: bool = False, enable_turn_audio: bool = False,
**kwargs, **kwargs,
): ):
"""Initialize the audio buffer processor.
Args:
sample_rate: Desired output sample rate. If None, uses source rate.
num_channels: Number of channels (1 for mono, 2 for stereo). Defaults to 1.
buffer_size: Size of buffer before triggering events. 0 for no buffering.
user_continuous_stream: Deprecated parameter for backwards compatibility.
enable_turn_audio: Whether turn audio event handlers should be triggered.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._init_sample_rate = sample_rate self._init_sample_rate = sample_rate
self._sample_rate = 0 self._sample_rate = 0
@@ -105,7 +116,7 @@ class AudioBufferProcessor(FrameProcessor):
"""Current sample rate of the audio processor. """Current sample rate of the audio processor.
Returns: Returns:
int: The sample rate in Hz The sample rate in Hz.
""" """
return self._sample_rate return self._sample_rate
@@ -114,7 +125,7 @@ class AudioBufferProcessor(FrameProcessor):
"""Number of channels in the audio output. """Number of channels in the audio output.
Returns: Returns:
int: Number of channels (1 for mono, 2 for stereo) Number of channels (1 for mono, 2 for stereo).
""" """
return self._num_channels return self._num_channels
@@ -122,7 +133,7 @@ class AudioBufferProcessor(FrameProcessor):
"""Check if both user and bot audio buffers contain data. """Check if both user and bot audio buffers contain data.
Returns: Returns:
bool: True if both buffers contain audio data True if both buffers contain audio data.
""" """
return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio( return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio(
self._bot_audio_buffer self._bot_audio_buffer
@@ -135,7 +146,7 @@ class AudioBufferProcessor(FrameProcessor):
on the left channel and bot audio on the right channel. on the left channel and bot audio on the right channel.
Returns: Returns:
bytes: Mixed audio data Mixed audio data as bytes.
""" """
if self._num_channels == 1: if self._num_channels == 1:
return mix_audio(bytes(self._user_audio_buffer), bytes(self._bot_audio_buffer)) return mix_audio(bytes(self._user_audio_buffer), bytes(self._bot_audio_buffer))
@@ -163,7 +174,12 @@ class AudioBufferProcessor(FrameProcessor):
self._recording = False self._recording = False
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming audio frames and manage audio buffers.""" """Process incoming audio frames and manage audio buffers.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# Update output sample rate if necessary. # Update output sample rate if necessary.
@@ -181,10 +197,12 @@ class AudioBufferProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
def _update_sample_rate(self, frame: StartFrame): def _update_sample_rate(self, frame: StartFrame):
"""Update the sample rate from the start frame."""
self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
self._audio_buffer_size_1s = self._sample_rate * 2 self._audio_buffer_size_1s = self._sample_rate * 2
async def _process_recording(self, frame: Frame): async def _process_recording(self, frame: Frame):
"""Process audio frames for recording."""
if isinstance(frame, InputAudioRawFrame): if isinstance(frame, InputAudioRawFrame):
# Add silence if we need to. # Add silence if we need to.
silence = self._compute_silence(self._last_user_frame_at) silence = self._compute_silence(self._last_user_frame_at)
@@ -208,6 +226,7 @@ class AudioBufferProcessor(FrameProcessor):
await self._call_on_audio_data_handler() await self._call_on_audio_data_handler()
async def _process_turn_recording(self, frame: Frame): async def _process_turn_recording(self, frame: Frame):
"""Process frames for turn-based audio recording."""
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, UserStartedSpeakingFrame):
self._user_speaking = True self._user_speaking = True
elif isinstance(frame, UserStoppedSpeakingFrame): elif isinstance(frame, UserStoppedSpeakingFrame):
@@ -242,6 +261,7 @@ class AudioBufferProcessor(FrameProcessor):
self._bot_turn_audio_buffer += resampled self._bot_turn_audio_buffer += resampled
async def _call_on_audio_data_handler(self): async def _call_on_audio_data_handler(self):
"""Call the audio data event handlers with buffered audio."""
if not self.has_audio() or not self._recording: if not self.has_audio() or not self._recording:
return return
@@ -263,23 +283,28 @@ class AudioBufferProcessor(FrameProcessor):
self._reset_audio_buffers() self._reset_audio_buffers()
def _buffer_has_audio(self, buffer: bytearray) -> bool: def _buffer_has_audio(self, buffer: bytearray) -> bool:
"""Check if a buffer contains audio data."""
return buffer is not None and len(buffer) > 0 return buffer is not None and len(buffer) > 0
def _reset_recording(self): def _reset_recording(self):
"""Reset recording state and buffers."""
self._reset_audio_buffers() self._reset_audio_buffers()
self._last_user_frame_at = time.time() self._last_user_frame_at = time.time()
self._last_bot_frame_at = time.time() self._last_bot_frame_at = time.time()
def _reset_audio_buffers(self): def _reset_audio_buffers(self):
"""Reset all audio buffers to empty state."""
self._user_audio_buffer = bytearray() self._user_audio_buffer = bytearray()
self._bot_audio_buffer = bytearray() self._bot_audio_buffer = bytearray()
self._user_turn_audio_buffer = bytearray() self._user_turn_audio_buffer = bytearray()
self._bot_turn_audio_buffer = bytearray() self._bot_turn_audio_buffer = bytearray()
async def _resample_audio(self, frame: AudioRawFrame) -> bytes: async def _resample_audio(self, frame: AudioRawFrame) -> bytes:
"""Resample audio frame to the target sample rate."""
return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate) return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate)
def _compute_silence(self, from_time: float) -> bytes: def _compute_silence(self, from_time: float) -> bytes:
"""Compute silence to insert based on time gap."""
quiet_time = time.time() - from_time quiet_time = time.time() - from_time
# We should get audio frames very frequently. We introduce silence only # We should get audio frames very frequently. We introduce silence only
# if there's a big enough gap of 1s. # if there's a big enough gap of 1s.

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Consumer processor for consuming frames from ProducerProcessor queues."""
import asyncio import asyncio
from typing import Awaitable, Callable, Optional from typing import Awaitable, Callable, Optional
@@ -14,11 +16,11 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
class ConsumerProcessor(FrameProcessor): class ConsumerProcessor(FrameProcessor):
"""This class passes-through frames and also consumes frames from a """Frame processor that consumes frames from a ProducerProcessor's queue.
producer's queue. When a frame from a producer queue is received it will be
pushed to the specified direction. The frames can be transformed into a
different type of frame before being pushed.
This processor passes through frames normally while also consuming frames
from a ProducerProcessor's queue. When frames are received from the producer
queue, they are optionally transformed and pushed in the specified direction.
""" """
def __init__( def __init__(
@@ -29,6 +31,14 @@ class ConsumerProcessor(FrameProcessor):
direction: FrameDirection = FrameDirection.DOWNSTREAM, direction: FrameDirection = FrameDirection.DOWNSTREAM,
**kwargs, **kwargs,
): ):
"""Initialize the consumer processor.
Args:
producer: The producer processor to consume frames from.
transformer: Function to transform frames before pushing. Defaults to identity_transformer.
direction: Direction to push consumed frames. Defaults to DOWNSTREAM.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._transformer = transformer self._transformer = transformer
self._direction = direction self._direction = direction
@@ -36,6 +46,12 @@ class ConsumerProcessor(FrameProcessor):
self._consumer_task: Optional[asyncio.Task] = None self._consumer_task: Optional[asyncio.Task] = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle lifecycle events.
Args:
frame: The frame to process.
direction: The direction the frame is traveling.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
@@ -48,19 +64,23 @@ class ConsumerProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def _start(self, _: StartFrame): async def _start(self, _: StartFrame):
"""Start the consumer task and register with the producer."""
if not self._consumer_task: if not self._consumer_task:
self._queue: WatchdogQueue = self._producer.add_consumer() self._queue: WatchdogQueue = self._producer.add_consumer()
self._consumer_task = self.create_task(self._consumer_task_handler()) self._consumer_task = self.create_task(self._consumer_task_handler())
async def _stop(self, _: EndFrame): async def _stop(self, _: EndFrame):
"""Stop the consumer task."""
if self._consumer_task: if self._consumer_task:
await self.cancel_task(self._consumer_task) await self.cancel_task(self._consumer_task)
async def _cancel(self, _: CancelFrame): async def _cancel(self, _: CancelFrame):
"""Cancel the consumer task."""
if self._consumer_task: if self._consumer_task:
await self.cancel_task(self._consumer_task) await self.cancel_task(self._consumer_task)
async def _consumer_task_handler(self): async def _consumer_task_handler(self):
"""Handle consuming frames from the producer queue."""
while True: while True:
frame = await self._queue.get() frame = await self._queue.get()
new_frame = await self._transformer(frame) new_frame = await self._transformer(frame)

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Frame filtering processor for the Pipecat framework."""
from typing import Tuple, Type from typing import Tuple, Type
from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.frames.frames import EndFrame, Frame, SystemFrame
@@ -11,7 +13,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FrameFilter(FrameProcessor): class FrameFilter(FrameProcessor):
"""A frame processor that filters frames based on their types.
This processor acts as a selective gate in the pipeline, allowing only
frames of specified types to pass through. System and end frames are
automatically allowed to pass through to maintain pipeline integrity.
"""
def __init__(self, types: Tuple[Type[Frame], ...]): def __init__(self, types: Tuple[Type[Frame], ...]):
"""Initialize the frame filter.
Args:
types: Tuple of frame types that should be allowed to pass through
the filter. All other frame types (except SystemFrame and
EndFrame) will be blocked.
"""
super().__init__() super().__init__()
self._types = types self._types = types
@@ -20,12 +36,19 @@ class FrameFilter(FrameProcessor):
# #
def _should_passthrough_frame(self, frame): def _should_passthrough_frame(self, frame):
"""Determine if a frame should pass through the filter."""
if isinstance(frame, self._types): if isinstance(frame, self._types):
return True return True
return isinstance(frame, (EndFrame, SystemFrame)) return isinstance(frame, (EndFrame, SystemFrame))
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process an incoming frame and conditionally pass it through.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if self._should_passthrough_frame(frame): if self._should_passthrough_frame(frame):

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Function-based frame filtering for Pipecat pipelines.
This module provides a processor that filters frames based on a custom function,
allowing for flexible frame filtering logic in processing pipelines.
"""
from typing import Awaitable, Callable from typing import Awaitable, Callable
from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.frames.frames import EndFrame, Frame, SystemFrame
@@ -11,11 +17,26 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class FunctionFilter(FrameProcessor): class FunctionFilter(FrameProcessor):
"""A frame processor that filters frames using a custom function.
This processor allows frames to pass through based on the result of a
user-provided filter function. System and end frames always pass through
regardless of the filter result.
"""
def __init__( def __init__(
self, self,
filter: Callable[[Frame], Awaitable[bool]], filter: Callable[[Frame], Awaitable[bool]],
direction: FrameDirection = FrameDirection.DOWNSTREAM, direction: FrameDirection = FrameDirection.DOWNSTREAM,
): ):
"""Initialize the function filter.
Args:
filter: An async function that takes a Frame and returns True if the
frame should pass through, False otherwise.
direction: The direction to apply filtering. Only frames moving in
this direction will be filtered. Defaults to DOWNSTREAM.
"""
super().__init__() super().__init__()
self._filter = filter self._filter = filter
self._direction = direction self._direction = direction
@@ -27,9 +48,18 @@ class FunctionFilter(FrameProcessor):
# Ignore system frames, end frames and frames that are not following the # Ignore system frames, end frames and frames that are not following the
# direction of this gate # direction of this gate
def _should_passthrough_frame(self, frame, direction): def _should_passthrough_frame(self, frame, direction):
"""Check if a frame should pass through without filtering."""
# Ignore system frames, end frames and frames that are not following the
# direction of this gate
return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame through the filter.
Args:
frame: The frame to process.
direction: The direction the frame is moving in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
passthrough = self._should_passthrough_frame(frame, direction) passthrough = self._should_passthrough_frame(frame, direction)

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Identity filter for transparent frame passthrough.
This module provides a simple passthrough filter that forwards all frames
without modification, useful for testing and pipeline composition.
"""
from pipecat.frames.frames import Frame from pipecat.frames.frames import Frame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -14,10 +20,14 @@ class IdentityFilter(FrameProcessor):
This filter acts as a transparent passthrough, allowing all frames to flow This filter acts as a transparent passthrough, allowing all frames to flow
through unchanged. It can be useful when testing `ParallelPipeline` to through unchanged. It can be useful when testing `ParallelPipeline` to
create pipelines that pass through frames (no frames should be repeated). create pipelines that pass through frames (no frames should be repeated).
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the identity filter.
Args:
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
# #
@@ -25,6 +35,11 @@ class IdentityFilter(FrameProcessor):
# #
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process an incoming frame by passing it through unchanged.""" """Process an incoming frame by passing it through unchanged.
Args:
frame: The frame to process and forward.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
await self.push_frame(frame, direction) await self.push_frame(frame, direction)

View File

@@ -4,14 +4,31 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Null filter processor for blocking frame transmission.
This module provides a frame processor that blocks all frames except
system and end frames, useful for testing or temporarily stopping
frame flow in a pipeline.
"""
from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.frames.frames import EndFrame, Frame, SystemFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class NullFilter(FrameProcessor): class NullFilter(FrameProcessor):
"""This filter doesn't allow passing any frames up or downstream.""" """A filter that blocks all frames except system and end frames.
This processor acts as a null filter, preventing frames from passing
through the pipeline while still allowing essential system and end
frames to maintain proper pipeline operation.
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the null filter.
Args:
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
# #
@@ -19,6 +36,12 @@ class NullFilter(FrameProcessor):
# #
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames, only allowing system and end frames through.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, (SystemFrame, EndFrame)): if isinstance(frame, (SystemFrame, EndFrame)):

View File

@@ -39,12 +39,17 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class STTMuteStrategy(Enum): class STTMuteStrategy(Enum):
"""Strategies determining when STT should be muted. """Strategies determining when STT should be muted.
Attributes: Each strategy defines different conditions under which speech-to-text
FIRST_SPEECH: Mute only during first detected bot speech processing should be temporarily disabled to prevent unwanted audio
MUTE_UNTIL_FIRST_BOT_COMPLETE: Start muted and remain muted until first bot speech completes processing during specific conversation states.
FUNCTION_CALL: Mute during function calls
ALWAYS: Mute during all bot speech Parameters:
CUSTOM: Allow custom logic via callback FIRST_SPEECH: Mute STT until the first bot speech is detected.
MUTE_UNTIL_FIRST_BOT_COMPLETE: Mute STT until the first bot completes speaking,
regardless of whether it is the first speech.
FUNCTION_CALL: Mute STT during function calls to prevent interruptions.
ALWAYS: Always mute STT when the bot is speaking.
CUSTOM: Use a custom callback to determine muting logic dynamically.
""" """
FIRST_SPEECH = "first_speech" FIRST_SPEECH = "first_speech"
@@ -58,10 +63,15 @@ class STTMuteStrategy(Enum):
class STTMuteConfig: class STTMuteConfig:
"""Configuration for STT muting behavior. """Configuration for STT muting behavior.
Args: Defines which muting strategies to apply and provides optional custom
strategies: Set of muting strategies to apply callback for advanced muting logic. Multiple strategies can be combined
to create sophisticated muting behavior.
Parameters:
strategies: Set of muting strategies to apply simultaneously.
should_mute_callback: Optional callback for custom muting logic. should_mute_callback: Optional callback for custom muting logic.
Only required when using STTMuteStrategy.CUSTOM Only required when using STTMuteStrategy.CUSTOM. Called with
the STTMuteFilter instance to determine muting state.
Note: Note:
MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together
@@ -69,10 +79,14 @@ class STTMuteConfig:
""" """
strategies: set[STTMuteStrategy] strategies: set[STTMuteStrategy]
# Optional callback for custom muting logic
should_mute_callback: Optional[Callable[["STTMuteFilter"], Awaitable[bool]]] = None should_mute_callback: Optional[Callable[["STTMuteFilter"], Awaitable[bool]]] = None
def __post_init__(self): def __post_init__(self):
"""Validate configuration after initialization.
Raises:
ValueError: If incompatible strategies are used together.
"""
if ( if (
STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE in self.strategies STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE in self.strategies
and STTMuteStrategy.FIRST_SPEECH in self.strategies and STTMuteStrategy.FIRST_SPEECH in self.strategies
@@ -86,15 +100,18 @@ class STTMuteFilter(FrameProcessor):
"""A processor that handles STT muting and interruption control. """A processor that handles STT muting and interruption control.
This processor combines STT muting and interruption control as a coordinated This processor combines STT muting and interruption control as a coordinated
feature. When STT is muted, interruptions are automatically disabled. feature. When STT is muted, interruptions are automatically disabled by
suppressing VAD-related frames. This prevents unwanted speech detection
Args: during bot speech, function calls, or other specified conditions.
config: Configuration specifying muting strategies
stt_service: STT service instance (deprecated, will be removed in future version)
**kwargs: Additional arguments passed to parent class
""" """
def __init__(self, *, config: STTMuteConfig, **kwargs): def __init__(self, *, config: STTMuteConfig, **kwargs):
"""Initialize the STT mute filter.
Args:
config: Configuration specifying muting strategies and behavior.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._config = config self._config = config
self._first_speech_handled = False self._first_speech_handled = False
@@ -104,18 +121,22 @@ class STTMuteFilter(FrameProcessor):
@property @property
def is_muted(self) -> bool: def is_muted(self) -> bool:
"""Returns whether STT is currently muted.""" """Check if STT is currently muted.
Returns:
True if STT is currently muted and audio frames are being suppressed.
"""
return self._is_muted return self._is_muted
async def _handle_mute_state(self, should_mute: bool): async def _handle_mute_state(self, should_mute: bool):
"""Handles both STT muting and interruption control.""" """Handle STT muting and interruption control state changes."""
if should_mute != self.is_muted: if should_mute != self.is_muted:
logger.debug(f"STTMuteFilter {'muting' if should_mute else 'unmuting'}") logger.debug(f"STTMuteFilter {'muting' if should_mute else 'unmuting'}")
self._is_muted = should_mute self._is_muted = should_mute
await self.push_frame(STTMuteFrame(mute=should_mute)) await self.push_frame(STTMuteFrame(mute=should_mute))
async def _should_mute(self) -> bool: async def _should_mute(self) -> bool:
"""Determines if STT should be muted based on current state and strategy.""" """Determine if STT should be muted based on current state and strategies."""
for strategy in self._config.strategies: for strategy in self._config.strategies:
match strategy: match strategy:
case STTMuteStrategy.FUNCTION_CALL: case STTMuteStrategy.FUNCTION_CALL:
@@ -144,7 +165,16 @@ class STTMuteFilter(FrameProcessor):
return False return False
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Processes incoming frames and manages muting state.""" """Process incoming frames and manage muting state.
Monitors conversation state through frame types and applies muting
strategies accordingly. Suppresses VAD-related frames when muted
while allowing other frames to pass through.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# Determine if we need to change mute state based on frame type # Determine if we need to change mute state based on frame type

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Wake phrase detection filter for Pipecat transcription processing.
This module provides a frame processor that filters transcription frames,
only allowing them through after wake phrases have been detected. Includes
keepalive functionality to maintain conversation flow after wake detection.
"""
import re import re
import time import time
from enum import Enum from enum import Enum
@@ -16,23 +23,53 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class WakeCheckFilter(FrameProcessor): class WakeCheckFilter(FrameProcessor):
"""This filter looks for wake phrases in the transcription frames and only passes through frames """Frame processor that filters transcription frames based on wake phrase detection.
after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief
period of continued conversation after a wake phrase has been detected. This filter monitors transcription frames for configured wake phrases and only
passes frames through after a wake phrase has been detected. Maintains a
keepalive timeout to allow continued conversation after wake detection.
""" """
class WakeState(Enum): class WakeState(Enum):
"""Enumeration of wake detection states.
Parameters:
IDLE: No wake phrase detected, filtering active.
AWAKE: Wake phrase detected, allowing frames through.
"""
IDLE = 1 IDLE = 1
AWAKE = 2 AWAKE = 2
class ParticipantState: class ParticipantState:
"""State tracking for individual participants.
Parameters:
participant_id: Unique identifier for the participant.
state: Current wake state (IDLE or AWAKE).
wake_timer: Timestamp of last wake phrase detection.
accumulator: Accumulated text for wake phrase matching.
"""
def __init__(self, participant_id: str): def __init__(self, participant_id: str):
"""Initialize participant state.
Args:
participant_id: Unique identifier for the participant.
"""
self.participant_id = participant_id self.participant_id = participant_id
self.state = WakeCheckFilter.WakeState.IDLE self.state = WakeCheckFilter.WakeState.IDLE
self.wake_timer = 0.0 self.wake_timer = 0.0
self.accumulator = "" self.accumulator = ""
def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3): def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3):
"""Initialize the wake phrase filter.
Args:
wake_phrases: List of wake phrases to detect in transcriptions.
keepalive_timeout: Duration in seconds to keep passing frames after
wake detection. Defaults to 3 seconds.
"""
super().__init__() super().__init__()
self._participant_states = {} self._participant_states = {}
self._keepalive_timeout = keepalive_timeout self._keepalive_timeout = keepalive_timeout
@@ -44,6 +81,12 @@ class WakeCheckFilter(FrameProcessor):
self._wake_patterns.append(pattern) self._wake_patterns.append(pattern)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames, filtering transcriptions based on wake detection.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
try: try:

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Wake notifier filter for conditional frame-based notifications."""
from typing import Awaitable, Callable, Tuple, Type from typing import Awaitable, Callable, Tuple, Type
from pipecat.frames.frames import Frame from pipecat.frames.frames import Frame
@@ -12,10 +14,11 @@ from pipecat.sync.base_notifier import BaseNotifier
class WakeNotifierFilter(FrameProcessor): class WakeNotifierFilter(FrameProcessor):
"""This processor expects a list of frame types and will execute a given """Frame processor that conditionally triggers notifications based on frame types and filters.
callback predicate when a frame of any of those type is being processed. If
the callback returns true the notifier will be notified.
This processor monitors frames of specified types and executes a callback predicate
when such frames are processed. If the callback returns True, the associated
notifier is triggered, allowing for conditional wake-up or notification scenarios.
""" """
def __init__( def __init__(
@@ -26,12 +29,27 @@ class WakeNotifierFilter(FrameProcessor):
filter: Callable[[Frame], Awaitable[bool]], filter: Callable[[Frame], Awaitable[bool]],
**kwargs, **kwargs,
): ):
"""Initialize the wake notifier filter.
Args:
notifier: The notifier to trigger when conditions are met.
types: Tuple of frame types to monitor for potential notifications.
filter: Async callback that determines whether to trigger notification.
Should return True to trigger notification, False otherwise.
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._notifier = notifier self._notifier = notifier
self._types = types self._types = types
self._filter = filter self._filter = filter
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and conditionally trigger notifications.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, self._types) and await self._filter(frame): if isinstance(frame, self._types) and await self._filter(frame):

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Frame processing pipeline infrastructure for Pipecat.
This module provides the core frame processing system that enables building
audio/video processing pipelines. It includes frame processors, pipeline
management, and frame flow control mechanisms.
"""
import asyncio import asyncio
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum from enum import Enum
@@ -36,12 +43,28 @@ from pipecat.utils.base_object import BaseObject
class FrameDirection(Enum): class FrameDirection(Enum):
"""Direction of frame flow in the processing pipeline.
Parameters:
DOWNSTREAM: Frames flowing from input to output.
UPSTREAM: Frames flowing back from output to input.
"""
DOWNSTREAM = 1 DOWNSTREAM = 1
UPSTREAM = 2 UPSTREAM = 2
@dataclass @dataclass
class FrameProcessorSetup: class FrameProcessorSetup:
"""Configuration parameters for frame processor initialization.
Parameters:
clock: The clock instance for timing operations.
task_manager: The task manager for handling async operations.
observer: Optional observer for monitoring frame processing events.
watchdog_timers_enabled: Whether to enable watchdog timers by default.
"""
clock: BaseClock clock: BaseClock
task_manager: BaseTaskManager task_manager: BaseTaskManager
observer: Optional[BaseObserver] = None observer: Optional[BaseObserver] = None
@@ -49,6 +72,14 @@ class FrameProcessorSetup:
class FrameProcessor(BaseObject): class FrameProcessor(BaseObject):
"""Base class for all frame processors in the pipeline.
Frame processors are the building blocks of Pipecat pipelines. They receive
frames, process them, and pass them to the next processor in the chain.
Each processor runs in its own task and can be linked to form complex
processing pipelines.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -59,6 +90,16 @@ class FrameProcessor(BaseObject):
watchdog_timeout_secs: Optional[float] = None, watchdog_timeout_secs: Optional[float] = None,
**kwargs, **kwargs,
): ):
"""Initialize the frame processor.
Args:
name: Optional name for this processor instance.
enable_watchdog_logging: Whether to enable watchdog logging for tasks.
enable_watchdog_timers: Whether to enable watchdog timers for tasks.
metrics: Optional metrics collector for this processor.
watchdog_timeout_secs: Timeout in seconds for watchdog operations.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(name=name) super().__init__(name=name)
self._parent: Optional["FrameProcessor"] = None self._parent: Optional["FrameProcessor"] = None
self._prev: Optional["FrameProcessor"] = None self._prev: Optional["FrameProcessor"] = None
@@ -118,77 +159,145 @@ class FrameProcessor(BaseObject):
@property @property
def id(self) -> int: def id(self) -> int:
"""Get the unique identifier for this processor.
Returns:
The unique integer ID of this processor.
"""
return self._id return self._id
@property @property
def name(self) -> str: def name(self) -> str:
"""Get the name of this processor.
Returns:
The name of this processor instance.
"""
return self._name return self._name
@property @property
def interruptions_allowed(self): def interruptions_allowed(self):
"""Check if interruptions are allowed for this processor.
Returns:
True if interruptions are allowed.
"""
return self._allow_interruptions return self._allow_interruptions
@property @property
def metrics_enabled(self): def metrics_enabled(self):
"""Check if metrics collection is enabled.
Returns:
True if metrics collection is enabled.
"""
return self._enable_metrics return self._enable_metrics
@property @property
def usage_metrics_enabled(self): def usage_metrics_enabled(self):
"""Check if usage metrics collection is enabled.
Returns:
True if usage metrics collection is enabled.
"""
return self._enable_usage_metrics return self._enable_usage_metrics
@property @property
def report_only_initial_ttfb(self): def report_only_initial_ttfb(self):
"""Check if only initial TTFB should be reported.
Returns:
True if only initial time-to-first-byte should be reported.
"""
return self._report_only_initial_ttfb return self._report_only_initial_ttfb
@property @property
def interruption_strategies(self) -> Sequence[BaseInterruptionStrategy]: def interruption_strategies(self) -> Sequence[BaseInterruptionStrategy]:
"""Get the interruption strategies for this processor.
Returns:
Sequence of interruption strategies.
"""
return self._interruption_strategies return self._interruption_strategies
@property @property
def task_manager(self) -> BaseTaskManager: def task_manager(self) -> BaseTaskManager:
"""Get the task manager for this processor.
Returns:
The task manager instance.
Raises:
Exception: If the task manager is not initialized.
"""
if not self._task_manager: if not self._task_manager:
raise Exception(f"{self} TaskManager is still not initialized.") raise Exception(f"{self} TaskManager is still not initialized.")
return self._task_manager return self._task_manager
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this processor can generate metrics.
Returns:
True if this processor can generate metrics.
"""
return False return False
def set_core_metrics_data(self, data: MetricsData): def set_core_metrics_data(self, data: MetricsData):
"""Set core metrics data for this processor.
Args:
data: The metrics data to set.
"""
self._metrics.set_core_metrics_data(data) self._metrics.set_core_metrics_data(data)
async def start_ttfb_metrics(self): async def start_ttfb_metrics(self):
"""Start time-to-first-byte metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled: if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb) await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb)
async def stop_ttfb_metrics(self): async def stop_ttfb_metrics(self):
"""Stop time-to-first-byte metrics collection and push results."""
if self.can_generate_metrics() and self.metrics_enabled: if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_ttfb_metrics() frame = await self._metrics.stop_ttfb_metrics()
if frame: if frame:
await self.push_frame(frame) await self.push_frame(frame)
async def start_processing_metrics(self): async def start_processing_metrics(self):
"""Start processing metrics collection."""
if self.can_generate_metrics() and self.metrics_enabled: if self.can_generate_metrics() and self.metrics_enabled:
await self._metrics.start_processing_metrics() await self._metrics.start_processing_metrics()
async def stop_processing_metrics(self): async def stop_processing_metrics(self):
"""Stop processing metrics collection and push results."""
if self.can_generate_metrics() and self.metrics_enabled: if self.can_generate_metrics() and self.metrics_enabled:
frame = await self._metrics.stop_processing_metrics() frame = await self._metrics.stop_processing_metrics()
if frame: if frame:
await self.push_frame(frame) await self.push_frame(frame)
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
"""Start LLM usage metrics collection.
Args:
tokens: Token usage information for the LLM.
"""
if self.can_generate_metrics() and self.usage_metrics_enabled: if self.can_generate_metrics() and self.usage_metrics_enabled:
frame = await self._metrics.start_llm_usage_metrics(tokens) frame = await self._metrics.start_llm_usage_metrics(tokens)
if frame: if frame:
await self.push_frame(frame) await self.push_frame(frame)
async def start_tts_usage_metrics(self, text: str): async def start_tts_usage_metrics(self, text: str):
"""Start TTS usage metrics collection.
Args:
text: The text being processed by TTS.
"""
if self.can_generate_metrics() and self.usage_metrics_enabled: if self.can_generate_metrics() and self.usage_metrics_enabled:
frame = await self._metrics.start_tts_usage_metrics(text) frame = await self._metrics.start_tts_usage_metrics(text)
if frame: if frame:
await self.push_frame(frame) await self.push_frame(frame)
async def stop_all_metrics(self): async def stop_all_metrics(self):
"""Stop all active metrics collection."""
await self.stop_ttfb_metrics() await self.stop_ttfb_metrics()
await self.stop_processing_metrics() await self.stop_processing_metrics()
@@ -201,6 +310,18 @@ class FrameProcessor(BaseObject):
enable_watchdog_timers: Optional[bool] = None, enable_watchdog_timers: Optional[bool] = None,
watchdog_timeout_secs: Optional[float] = None, watchdog_timeout_secs: Optional[float] = None,
) -> asyncio.Task: ) -> asyncio.Task:
"""Create a new task managed by this processor.
Args:
coroutine: The coroutine to run in the task.
name: Optional name for the task.
enable_watchdog_logging: Whether to enable watchdog logging.
enable_watchdog_timers: Whether to enable watchdog timers.
watchdog_timeout_secs: Timeout in seconds for watchdog operations.
Returns:
The created asyncio task.
"""
if name: if name:
name = f"{self}::{name}" name = f"{self}::{name}"
else: else:
@@ -222,15 +343,33 @@ class FrameProcessor(BaseObject):
) )
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
"""Cancel a task managed by this processor.
Args:
task: The task to cancel.
timeout: Optional timeout for task cancellation.
"""
await self.task_manager.cancel_task(task, timeout) await self.task_manager.cancel_task(task, timeout)
async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None): async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None):
"""Wait for a task to complete.
Args:
task: The task to wait for.
timeout: Optional timeout for waiting.
"""
await self.task_manager.wait_for_task(task, timeout) await self.task_manager.wait_for_task(task, timeout)
def reset_watchdog(self): def reset_watchdog(self):
"""Reset the watchdog timer for the current task."""
self.task_manager.task_reset_watchdog() self.task_manager.task_reset_watchdog()
async def setup(self, setup: FrameProcessorSetup): async def setup(self, setup: FrameProcessorSetup):
"""Set up the processor with required components.
Args:
setup: Configuration object containing setup parameters.
"""
self._clock = setup.clock self._clock = setup.clock
self._task_manager = setup.task_manager self._task_manager = setup.task_manager
self._observer = setup.observer self._observer = setup.observer
@@ -243,6 +382,7 @@ class FrameProcessor(BaseObject):
await self._metrics.setup(self._task_manager) await self._metrics.setup(self._task_manager)
async def cleanup(self): async def cleanup(self):
"""Clean up processor resources."""
await super().cleanup() await super().cleanup()
await self.__cancel_input_task() await self.__cancel_input_task()
await self.__cancel_push_task() await self.__cancel_push_task()
@@ -250,20 +390,48 @@ class FrameProcessor(BaseObject):
await self._metrics.cleanup() await self._metrics.cleanup()
def link(self, processor: "FrameProcessor"): def link(self, processor: "FrameProcessor"):
"""Link this processor to the next processor in the pipeline.
Args:
processor: The processor to link to.
"""
self._next = processor self._next = processor
processor._prev = self processor._prev = self
logger.debug(f"Linking {self} -> {self._next}") logger.debug(f"Linking {self} -> {self._next}")
def get_event_loop(self) -> asyncio.AbstractEventLoop: def get_event_loop(self) -> asyncio.AbstractEventLoop:
"""Get the event loop used by this processor.
Returns:
The asyncio event loop.
"""
return self.task_manager.get_event_loop() return self.task_manager.get_event_loop()
def set_parent(self, parent: "FrameProcessor"): def set_parent(self, parent: "FrameProcessor"):
"""Set the parent processor for this processor.
Args:
parent: The parent processor.
"""
self._parent = parent self._parent = parent
def get_parent(self) -> Optional["FrameProcessor"]: def get_parent(self) -> Optional["FrameProcessor"]:
"""Get the parent processor.
Returns:
The parent processor, or None if no parent is set.
"""
return self._parent return self._parent
def get_clock(self) -> BaseClock: def get_clock(self) -> BaseClock:
"""Get the clock used by this processor.
Returns:
The clock instance.
Raises:
Exception: If the clock is not initialized.
"""
if not self._clock: if not self._clock:
raise Exception(f"{self} Clock is still not initialized.") raise Exception(f"{self} Clock is still not initialized.")
return self._clock return self._clock
@@ -276,6 +444,13 @@ class FrameProcessor(BaseObject):
Callable[["FrameProcessor", Frame, FrameDirection], Awaitable[None]] Callable[["FrameProcessor", Frame, FrameDirection], Awaitable[None]]
] = None, ] = None,
): ):
"""Queue a frame for processing.
Args:
frame: The frame to queue.
direction: The direction of frame flow.
callback: Optional callback to call after processing.
"""
# If we are cancelling we don't want to process any other frame. # If we are cancelling we don't want to process any other frame.
if self._cancelling: if self._cancelling:
return return
@@ -288,15 +463,23 @@ class FrameProcessor(BaseObject):
await self.__input_queue.put((frame, direction, callback)) await self.__input_queue.put((frame, direction, callback))
async def pause_processing_frames(self): async def pause_processing_frames(self):
"""Pause processing of queued frames."""
logger.trace(f"{self}: pausing frame processing") logger.trace(f"{self}: pausing frame processing")
self.__should_block_frames = True self.__should_block_frames = True
async def resume_processing_frames(self): async def resume_processing_frames(self):
"""Resume processing of queued frames."""
logger.trace(f"{self}: resuming frame processing") logger.trace(f"{self}: resuming frame processing")
if self.__input_event: if self.__input_event:
self.__input_event.set() self.__input_event.set()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
await self.__start(frame) await self.__start(frame)
elif isinstance(frame, StartInterruptionFrame): elif isinstance(frame, StartInterruptionFrame):
@@ -312,9 +495,20 @@ class FrameProcessor(BaseObject):
await self.__resume(frame) await self.__resume(frame)
async def push_error(self, error: ErrorFrame): async def push_error(self, error: ErrorFrame):
"""Push an error frame upstream.
Args:
error: The error frame to push.
"""
await self.push_frame(error, FrameDirection.UPSTREAM) await self.push_frame(error, FrameDirection.UPSTREAM)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame to the next processor in the pipeline.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
if not self._check_started(frame): if not self._check_started(frame):
return return
@@ -324,6 +518,11 @@ class FrameProcessor(BaseObject):
await self.__push_queue.put((frame, direction)) await self.__push_queue.put((frame, direction))
async def __start(self, frame: StartFrame): async def __start(self, frame: StartFrame):
"""Handle the start frame to initialize processor state.
Args:
frame: The start frame containing initialization parameters.
"""
self.__started = True self.__started = True
self._allow_interruptions = frame.allow_interruptions self._allow_interruptions = frame.allow_interruptions
self._enable_metrics = frame.enable_metrics self._enable_metrics = frame.enable_metrics
@@ -334,15 +533,30 @@ class FrameProcessor(BaseObject):
self.__create_push_task() self.__create_push_task()
async def __cancel(self, frame: CancelFrame): async def __cancel(self, frame: CancelFrame):
"""Handle the cancel frame to stop processor operation.
Args:
frame: The cancel frame.
"""
self._cancelling = True self._cancelling = True
await self.__cancel_input_task() await self.__cancel_input_task()
await self.__cancel_push_task() await self.__cancel_push_task()
async def __pause(self, frame: FrameProcessorPauseFrame | FrameProcessorPauseUrgentFrame): async def __pause(self, frame: FrameProcessorPauseFrame | FrameProcessorPauseUrgentFrame):
"""Handle pause frame to pause processor operation.
Args:
frame: The pause frame.
"""
if frame.processor.name == self.name: if frame.processor.name == self.name:
await self.pause_processing_frames() await self.pause_processing_frames()
async def __resume(self, frame: FrameProcessorResumeFrame | FrameProcessorResumeUrgentFrame): async def __resume(self, frame: FrameProcessorResumeFrame | FrameProcessorResumeUrgentFrame):
"""Handle resume frame to resume processor operation.
Args:
frame: The resume frame.
"""
if frame.processor.name == self.name: if frame.processor.name == self.name:
await self.resume_processing_frames() await self.resume_processing_frames()
@@ -351,6 +565,7 @@ class FrameProcessor(BaseObject):
# #
async def _start_interruption(self): async def _start_interruption(self):
"""Start handling an interruption by canceling current tasks."""
try: try:
# Cancel the push frame task. This will stop pushing frames downstream. # Cancel the push frame task. This will stop pushing frames downstream.
await self.__cancel_push_task() await self.__cancel_push_task()
@@ -368,10 +583,17 @@ class FrameProcessor(BaseObject):
self.__create_push_task() self.__create_push_task()
async def _stop_interruption(self): async def _stop_interruption(self):
"""Stop handling an interruption."""
# Nothing to do right now. # Nothing to do right now.
pass pass
async def __internal_push_frame(self, frame: Frame, direction: FrameDirection): async def __internal_push_frame(self, frame: Frame, direction: FrameDirection):
"""Internal method to push frames to adjacent processors.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
try: try:
timestamp = self._clock.get_time() if self._clock else 0 timestamp = self._clock.get_time() if self._clock else 0
if direction == FrameDirection.DOWNSTREAM and self._next: if direction == FrameDirection.DOWNSTREAM and self._next:
@@ -404,11 +626,20 @@ class FrameProcessor(BaseObject):
await self.push_error(ErrorFrame(str(e))) await self.push_error(ErrorFrame(str(e)))
def _check_started(self, frame: Frame): def _check_started(self, frame: Frame):
"""Check if the processor has been started.
Args:
frame: The frame being processed.
Returns:
True if the processor has been started.
"""
if not self.__started: if not self.__started:
logger.error(f"{self} Trying to process {frame} but StartFrame not received yet") logger.error(f"{self} Trying to process {frame} but StartFrame not received yet")
return self.__started return self.__started
def __create_input_task(self): def __create_input_task(self):
"""Create the input processing task."""
if not self.__input_frame_task: if not self.__input_frame_task:
self.__should_block_frames = False self.__should_block_frames = False
if not self.__input_event: if not self.__input_event:
@@ -418,11 +649,13 @@ class FrameProcessor(BaseObject):
self.__input_frame_task = self.create_task(self.__input_frame_task_handler()) self.__input_frame_task = self.create_task(self.__input_frame_task_handler())
async def __cancel_input_task(self): async def __cancel_input_task(self):
"""Cancel the input processing task."""
if self.__input_frame_task: if self.__input_frame_task:
await self.cancel_task(self.__input_frame_task) await self.cancel_task(self.__input_frame_task)
self.__input_frame_task = None self.__input_frame_task = None
async def __input_frame_task_handler(self): async def __input_frame_task_handler(self):
"""Handle frames from the input queue."""
while True: while True:
if self.__should_block_frames and self.__input_event: if self.__should_block_frames and self.__input_event:
logger.trace(f"{self}: frame processing paused") logger.trace(f"{self}: frame processing paused")
@@ -445,16 +678,19 @@ class FrameProcessor(BaseObject):
self.__input_queue.task_done() self.__input_queue.task_done()
def __create_push_task(self): def __create_push_task(self):
"""Create the frame pushing task."""
if not self.__push_frame_task: if not self.__push_frame_task:
self.__push_queue = WatchdogQueue(self.task_manager) self.__push_queue = WatchdogQueue(self.task_manager)
self.__push_frame_task = self.create_task(self.__push_frame_task_handler()) self.__push_frame_task = self.create_task(self.__push_frame_task_handler())
async def __cancel_push_task(self): async def __cancel_push_task(self):
"""Cancel the frame pushing task."""
if self.__push_frame_task: if self.__push_frame_task:
await self.cancel_task(self.__push_frame_task) await self.cancel_task(self.__push_frame_task)
self.__push_frame_task = None self.__push_frame_task = None
async def __push_frame_task_handler(self): async def __push_frame_task_handler(self):
"""Handle frames from the push queue."""
while True: while True:
(frame, direction) = await self.__push_queue.get() (frame, direction) = await self.__push_queue.get()
await self.__internal_push_frame(frame, direction) await self.__internal_push_frame(frame, direction)

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Langchain integration processor for Pipecat."""
from typing import Optional, Union from typing import Optional, Union
from loguru import logger from loguru import logger
@@ -26,16 +28,40 @@ except ModuleNotFoundError as e:
class LangchainProcessor(FrameProcessor): class LangchainProcessor(FrameProcessor):
"""Processor that integrates Langchain runnables with Pipecat's frame pipeline.
This processor takes LLM message frames, extracts the latest user message,
and processes it through a Langchain runnable chain. The response is streamed
back as text frames with appropriate response markers.
"""
def __init__(self, chain: Runnable, transcript_key: str = "input"): def __init__(self, chain: Runnable, transcript_key: str = "input"):
"""Initialize the Langchain processor.
Args:
chain: The Langchain runnable to use for processing messages.
transcript_key: The key to use when passing input to the chain.
"""
super().__init__() super().__init__()
self._chain = chain self._chain = chain
self._transcript_key = transcript_key self._transcript_key = transcript_key
self._participant_id: Optional[str] = None self._participant_id: Optional[str] = None
def set_participant_id(self, participant_id: str): def set_participant_id(self, participant_id: str):
"""Set the participant ID for session tracking.
Args:
participant_id: The participant ID to use for session configuration.
"""
self._participant_id = participant_id self._participant_id = participant_id
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle LLM message frames.
Args:
frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, LLMMessagesFrame): if isinstance(frame, LLMMessagesFrame):
@@ -50,6 +76,14 @@ class LangchainProcessor(FrameProcessor):
@staticmethod @staticmethod
def __get_token_value(text: Union[str, AIMessageChunk]) -> str: def __get_token_value(text: Union[str, AIMessageChunk]) -> str:
"""Extract token value from various text types.
Args:
text: The text or message chunk to extract value from.
Returns:
The extracted string value.
"""
match text: match text:
case str(): case str():
return text return text
@@ -59,6 +93,7 @@ class LangchainProcessor(FrameProcessor):
return "" return ""
async def _ainvoke(self, text: str): async def _ainvoke(self, text: str):
"""Invoke the Langchain runnable with the provided text."""
logger.debug(f"Invoking chain with {text}") logger.debug(f"Invoking chain with {text}")
await self.push_frame(LLMFullResponseStartFrame()) await self.push_frame(LLMFullResponseStartFrame())
try: try:

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""RTVI (Real-Time Voice Interface) protocol implementation for Pipecat.
This module provides the RTVI protocol implementation for real-time voice interactions
between clients and AI agents. It includes message handling, action processing,
and frame observation for the RTVI protocol.
"""
import asyncio import asyncio
import base64 import base64
from dataclasses import dataclass from dataclasses import dataclass
@@ -79,6 +86,12 @@ ActionResult = Union[bool, int, float, str, list, dict]
class RTVIServiceOption(BaseModel): class RTVIServiceOption(BaseModel):
"""Configuration option for an RTVI service.
Defines a configurable option that can be set for an RTVI service,
including its name, type, and handler function.
"""
name: str name: str
type: Literal["bool", "number", "string", "array", "object"] type: Literal["bool", "number", "string", "array", "object"]
handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"], Awaitable[None]] = Field( handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"], Awaitable[None]] = Field(
@@ -87,11 +100,18 @@ class RTVIServiceOption(BaseModel):
class RTVIService(BaseModel): class RTVIService(BaseModel):
"""An RTVI service definition.
Represents a service that can be configured and used within the RTVI protocol,
containing a name and list of configurable options.
"""
name: str name: str
options: List[RTVIServiceOption] options: List[RTVIServiceOption]
_options_dict: Dict[str, RTVIServiceOption] = PrivateAttr(default={}) _options_dict: Dict[str, RTVIServiceOption] = PrivateAttr(default={})
def model_post_init(self, __context: Any) -> None: def model_post_init(self, __context: Any) -> None:
"""Initialize the options dictionary after model creation."""
self._options_dict = {} self._options_dict = {}
for option in self.options: for option in self.options:
self._options_dict[option.name] = option self._options_dict[option.name] = option
@@ -99,16 +119,32 @@ class RTVIService(BaseModel):
class RTVIActionArgumentData(BaseModel): class RTVIActionArgumentData(BaseModel):
"""Data for an RTVI action argument.
Contains the name and value of an argument passed to an RTVI action.
"""
name: str name: str
value: Any value: Any
class RTVIActionArgument(BaseModel): class RTVIActionArgument(BaseModel):
"""Definition of an RTVI action argument.
Specifies the name and expected type of an argument for an RTVI action.
"""
name: str name: str
type: Literal["bool", "number", "string", "array", "object"] type: Literal["bool", "number", "string", "array", "object"]
class RTVIAction(BaseModel): class RTVIAction(BaseModel):
"""An RTVI action definition.
Represents an action that can be executed within the RTVI protocol,
including its service, name, arguments, and handler function.
"""
service: str service: str
action: str action: str
arguments: List[RTVIActionArgument] = Field(default_factory=list) arguments: List[RTVIActionArgument] = Field(default_factory=list)
@@ -119,6 +155,7 @@ class RTVIAction(BaseModel):
_arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={}) _arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={})
def model_post_init(self, __context: Any) -> None: def model_post_init(self, __context: Any) -> None:
"""Initialize the arguments dictionary after model creation."""
self._arguments_dict = {} self._arguments_dict = {}
for arg in self.arguments: for arg in self.arguments:
self._arguments_dict[arg.name] = arg self._arguments_dict[arg.name] = arg
@@ -126,16 +163,31 @@ class RTVIAction(BaseModel):
class RTVIServiceOptionConfig(BaseModel): class RTVIServiceOptionConfig(BaseModel):
"""Configuration value for an RTVI service option.
Contains the name and value to set for a specific service option.
"""
name: str name: str
value: Any value: Any
class RTVIServiceConfig(BaseModel): class RTVIServiceConfig(BaseModel):
"""Configuration for an RTVI service.
Contains the service name and list of option configurations to apply.
"""
service: str service: str
options: List[RTVIServiceOptionConfig] options: List[RTVIServiceOptionConfig]
class RTVIConfig(BaseModel): class RTVIConfig(BaseModel):
"""Complete RTVI configuration.
Contains the full configuration for all RTVI services.
"""
config: List[RTVIServiceConfig] config: List[RTVIServiceConfig]
@@ -145,16 +197,31 @@ class RTVIConfig(BaseModel):
class RTVIUpdateConfig(BaseModel): class RTVIUpdateConfig(BaseModel):
"""Request to update RTVI configuration.
Contains new configuration settings and whether to interrupt the bot.
"""
config: List[RTVIServiceConfig] config: List[RTVIServiceConfig]
interrupt: bool = False interrupt: bool = False
class RTVIActionRunArgument(BaseModel): class RTVIActionRunArgument(BaseModel):
"""Argument for running an RTVI action.
Contains the name and value of an argument to pass to an action.
"""
name: str name: str
value: Any value: Any
class RTVIActionRun(BaseModel): class RTVIActionRun(BaseModel):
"""Request to run an RTVI action.
Contains the service, action name, and optional arguments.
"""
service: str service: str
action: str action: str
arguments: Optional[List[RTVIActionRunArgument]] = None arguments: Optional[List[RTVIActionRunArgument]] = None
@@ -162,11 +229,23 @@ class RTVIActionRun(BaseModel):
@dataclass @dataclass
class RTVIActionFrame(DataFrame): class RTVIActionFrame(DataFrame):
"""Frame containing an RTVI action to execute.
Parameters:
rtvi_action_run: The action to execute.
message_id: Optional message ID for response correlation.
"""
rtvi_action_run: RTVIActionRun rtvi_action_run: RTVIActionRun
message_id: Optional[str] = None message_id: Optional[str] = None
class RTVIMessage(BaseModel): class RTVIMessage(BaseModel):
"""Base RTVI message structure.
Represents the standard format for RTVI protocol messages.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: str type: str
id: str id: str
@@ -179,10 +258,20 @@ class RTVIMessage(BaseModel):
class RTVIErrorResponseData(BaseModel): class RTVIErrorResponseData(BaseModel):
"""Data for an RTVI error response.
Contains the error message to send back to the client.
"""
error: str error: str
class RTVIErrorResponse(BaseModel): class RTVIErrorResponse(BaseModel):
"""RTVI error response message.
Sent in response to a client request that resulted in an error.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["error-response"] = "error-response" type: Literal["error-response"] = "error-response"
id: str id: str
@@ -190,21 +279,41 @@ class RTVIErrorResponse(BaseModel):
class RTVIErrorData(BaseModel): class RTVIErrorData(BaseModel):
"""Data for an RTVI error event.
Contains error information including whether it's fatal.
"""
error: str error: str
fatal: bool fatal: bool
class RTVIError(BaseModel): class RTVIError(BaseModel):
"""RTVI error event message.
Sent when an error occurs that isn't in response to a specific request.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["error"] = "error" type: Literal["error"] = "error"
data: RTVIErrorData data: RTVIErrorData
class RTVIDescribeConfigData(BaseModel): class RTVIDescribeConfigData(BaseModel):
"""Data for describing available RTVI configuration.
Contains the list of available services and their options.
"""
config: List[RTVIService] config: List[RTVIService]
class RTVIDescribeConfig(BaseModel): class RTVIDescribeConfig(BaseModel):
"""Message describing available RTVI configuration.
Sent in response to a describe-config request.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["config-available"] = "config-available" type: Literal["config-available"] = "config-available"
id: str id: str
@@ -212,10 +321,20 @@ class RTVIDescribeConfig(BaseModel):
class RTVIDescribeActionsData(BaseModel): class RTVIDescribeActionsData(BaseModel):
"""Data for describing available RTVI actions.
Contains the list of available actions that can be executed.
"""
actions: List[RTVIAction] actions: List[RTVIAction]
class RTVIDescribeActions(BaseModel): class RTVIDescribeActions(BaseModel):
"""Message describing available RTVI actions.
Sent in response to a describe-actions request.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["actions-available"] = "actions-available" type: Literal["actions-available"] = "actions-available"
id: str id: str
@@ -223,6 +342,11 @@ class RTVIDescribeActions(BaseModel):
class RTVIConfigResponse(BaseModel): class RTVIConfigResponse(BaseModel):
"""Response containing current RTVI configuration.
Sent in response to a get-config request.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["config"] = "config" type: Literal["config"] = "config"
id: str id: str
@@ -230,10 +354,20 @@ class RTVIConfigResponse(BaseModel):
class RTVIActionResponseData(BaseModel): class RTVIActionResponseData(BaseModel):
"""Data for an RTVI action response.
Contains the result of executing an action.
"""
result: ActionResult result: ActionResult
class RTVIActionResponse(BaseModel): class RTVIActionResponse(BaseModel):
"""Response to an RTVI action execution.
Sent after successfully executing an action.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["action-response"] = "action-response" type: Literal["action-response"] = "action-response"
id: str id: str
@@ -241,11 +375,21 @@ class RTVIActionResponse(BaseModel):
class RTVIBotReadyData(BaseModel): class RTVIBotReadyData(BaseModel):
"""Data for bot ready notification.
Contains protocol version and initial configuration.
"""
version: str version: str
config: List[RTVIServiceConfig] config: List[RTVIServiceConfig]
class RTVIBotReady(BaseModel): class RTVIBotReady(BaseModel):
"""Message indicating bot is ready for interaction.
Sent after bot initialization is complete.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-ready"] = "bot-ready" type: Literal["bot-ready"] = "bot-ready"
id: str id: str
@@ -253,28 +397,53 @@ class RTVIBotReady(BaseModel):
class RTVILLMFunctionCallMessageData(BaseModel): class RTVILLMFunctionCallMessageData(BaseModel):
"""Data for LLM function call notification.
Contains function call details including name, ID, and arguments.
"""
function_name: str function_name: str
tool_call_id: str tool_call_id: str
args: Mapping[str, Any] args: Mapping[str, Any]
class RTVILLMFunctionCallMessage(BaseModel): class RTVILLMFunctionCallMessage(BaseModel):
"""Message notifying of an LLM function call.
Sent when the LLM makes a function call.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["llm-function-call"] = "llm-function-call" type: Literal["llm-function-call"] = "llm-function-call"
data: RTVILLMFunctionCallMessageData data: RTVILLMFunctionCallMessageData
class RTVILLMFunctionCallStartMessageData(BaseModel): class RTVILLMFunctionCallStartMessageData(BaseModel):
"""Data for LLM function call start notification.
Contains the function name being called.
"""
function_name: str function_name: str
class RTVILLMFunctionCallStartMessage(BaseModel): class RTVILLMFunctionCallStartMessage(BaseModel):
"""Message notifying that an LLM function call has started.
Sent when the LLM begins a function call.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["llm-function-call-start"] = "llm-function-call-start" type: Literal["llm-function-call-start"] = "llm-function-call-start"
data: RTVILLMFunctionCallStartMessageData data: RTVILLMFunctionCallStartMessageData
class RTVILLMFunctionCallResultData(BaseModel): class RTVILLMFunctionCallResultData(BaseModel):
"""Data for LLM function call result.
Contains function call details and result.
"""
function_name: str function_name: str
tool_call_id: str tool_call_id: str
arguments: dict arguments: dict
@@ -282,60 +451,103 @@ class RTVILLMFunctionCallResultData(BaseModel):
class RTVIBotLLMStartedMessage(BaseModel): class RTVIBotLLMStartedMessage(BaseModel):
"""Message indicating bot LLM processing has started."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-started"] = "bot-llm-started" type: Literal["bot-llm-started"] = "bot-llm-started"
class RTVIBotLLMStoppedMessage(BaseModel): class RTVIBotLLMStoppedMessage(BaseModel):
"""Message indicating bot LLM processing has stopped."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-stopped"] = "bot-llm-stopped" type: Literal["bot-llm-stopped"] = "bot-llm-stopped"
class RTVIBotTTSStartedMessage(BaseModel): class RTVIBotTTSStartedMessage(BaseModel):
"""Message indicating bot TTS processing has started."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-started"] = "bot-tts-started" type: Literal["bot-tts-started"] = "bot-tts-started"
class RTVIBotTTSStoppedMessage(BaseModel): class RTVIBotTTSStoppedMessage(BaseModel):
"""Message indicating bot TTS processing has stopped."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-stopped"] = "bot-tts-stopped" type: Literal["bot-tts-stopped"] = "bot-tts-stopped"
class RTVITextMessageData(BaseModel): class RTVITextMessageData(BaseModel):
"""Data for text-based RTVI messages.
Contains text content.
"""
text: str text: str
class RTVIBotTranscriptionMessage(BaseModel): class RTVIBotTranscriptionMessage(BaseModel):
"""Message containing bot transcription text.
Sent when the bot's speech is transcribed.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-transcription"] = "bot-transcription" type: Literal["bot-transcription"] = "bot-transcription"
data: RTVITextMessageData data: RTVITextMessageData
class RTVIBotLLMTextMessage(BaseModel): class RTVIBotLLMTextMessage(BaseModel):
"""Message containing bot LLM text output.
Sent when the bot's LLM generates text.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-llm-text"] = "bot-llm-text" type: Literal["bot-llm-text"] = "bot-llm-text"
data: RTVITextMessageData data: RTVITextMessageData
class RTVIBotTTSTextMessage(BaseModel): class RTVIBotTTSTextMessage(BaseModel):
"""Message containing bot TTS text output.
Sent when text is being processed by TTS.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-text"] = "bot-tts-text" type: Literal["bot-tts-text"] = "bot-tts-text"
data: RTVITextMessageData data: RTVITextMessageData
class RTVIAudioMessageData(BaseModel): class RTVIAudioMessageData(BaseModel):
"""Data for audio-based RTVI messages.
Contains audio data and metadata.
"""
audio: str audio: str
sample_rate: int sample_rate: int
num_channels: int num_channels: int
class RTVIBotTTSAudioMessage(BaseModel): class RTVIBotTTSAudioMessage(BaseModel):
"""Message containing bot TTS audio output.
Sent when the bot's TTS generates audio.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-tts-audio"] = "bot-tts-audio" type: Literal["bot-tts-audio"] = "bot-tts-audio"
data: RTVIAudioMessageData data: RTVIAudioMessageData
class RTVIUserTranscriptionMessageData(BaseModel): class RTVIUserTranscriptionMessageData(BaseModel):
"""Data for user transcription messages.
Contains transcription text and metadata.
"""
text: str text: str
user_id: str user_id: str
timestamp: str timestamp: str
@@ -343,44 +555,72 @@ class RTVIUserTranscriptionMessageData(BaseModel):
class RTVIUserTranscriptionMessage(BaseModel): class RTVIUserTranscriptionMessage(BaseModel):
"""Message containing user transcription.
Sent when user speech is transcribed.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-transcription"] = "user-transcription" type: Literal["user-transcription"] = "user-transcription"
data: RTVIUserTranscriptionMessageData data: RTVIUserTranscriptionMessageData
class RTVIUserLLMTextMessage(BaseModel): class RTVIUserLLMTextMessage(BaseModel):
"""Message containing user text input for LLM.
Sent when user text is processed by the LLM.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-llm-text"] = "user-llm-text" type: Literal["user-llm-text"] = "user-llm-text"
data: RTVITextMessageData data: RTVITextMessageData
class RTVIUserStartedSpeakingMessage(BaseModel): class RTVIUserStartedSpeakingMessage(BaseModel):
"""Message indicating user has started speaking."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-started-speaking"] = "user-started-speaking" type: Literal["user-started-speaking"] = "user-started-speaking"
class RTVIUserStoppedSpeakingMessage(BaseModel): class RTVIUserStoppedSpeakingMessage(BaseModel):
"""Message indicating user has stopped speaking."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["user-stopped-speaking"] = "user-stopped-speaking" type: Literal["user-stopped-speaking"] = "user-stopped-speaking"
class RTVIBotStartedSpeakingMessage(BaseModel): class RTVIBotStartedSpeakingMessage(BaseModel):
"""Message indicating bot has started speaking."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-started-speaking"] = "bot-started-speaking" type: Literal["bot-started-speaking"] = "bot-started-speaking"
class RTVIBotStoppedSpeakingMessage(BaseModel): class RTVIBotStoppedSpeakingMessage(BaseModel):
"""Message indicating bot has stopped speaking."""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["bot-stopped-speaking"] = "bot-stopped-speaking" type: Literal["bot-stopped-speaking"] = "bot-stopped-speaking"
class RTVIMetricsMessage(BaseModel): class RTVIMetricsMessage(BaseModel):
"""Message containing performance metrics.
Sent to provide performance and usage metrics.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["metrics"] = "metrics" type: Literal["metrics"] = "metrics"
data: Mapping[str, Any] data: Mapping[str, Any]
class RTVIServerMessage(BaseModel): class RTVIServerMessage(BaseModel):
"""Generic server message.
Used for custom server-to-client messages.
"""
label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL
type: Literal["server-message"] = "server-message" type: Literal["server-message"] = "server-message"
data: Any data: Any
@@ -388,28 +628,32 @@ class RTVIServerMessage(BaseModel):
@dataclass @dataclass
class RTVIServerMessageFrame(SystemFrame): class RTVIServerMessageFrame(SystemFrame):
"""A frame for sending server messages to the client.""" """A frame for sending server messages to the client.
Parameters:
data: The message data to send to the client.
"""
data: Any data: Any
def __str__(self): def __str__(self):
"""String representation of the RTVI server message frame."""
return f"{self.name}(data: {self.data})" return f"{self.name}(data: {self.data})"
@dataclass @dataclass
class RTVIObserverParams: class RTVIObserverParams:
""" """Parameters for configuring RTVI Observer behavior.
Parameters for configuring RTVI Observer behavior.
Attributes: Parameters:
bot_llm_enabled (bool): Indicates if the bot's LLM messages should be sent. bot_llm_enabled: Indicates if the bot's LLM messages should be sent.
bot_tts_enabled (bool): Indicates if the bot's TTS messages should be sent. bot_tts_enabled: Indicates if the bot's TTS messages should be sent.
bot_speaking_enabled (bool): Indicates if the bot's started/stopped speaking messages should be sent. bot_speaking_enabled: Indicates if the bot's started/stopped speaking messages should be sent.
user_llm_enabled (bool): Indicates if the user's LLM input messages should be sent. user_llm_enabled: Indicates if the user's LLM input messages should be sent.
user_speaking_enabled (bool): Indicates if the user's started/stopped speaking messages should be sent. user_speaking_enabled: Indicates if the user's started/stopped speaking messages should be sent.
user_transcription_enabled (bool): Indicates if user's transcription messages should be sent. user_transcription_enabled: Indicates if user's transcription messages should be sent.
metrics_enabled (bool): Indicates if metrics messages should be sent. metrics_enabled: Indicates if metrics messages should be sent.
errors_enabled (bool): Indicates if errors messages should be sent. errors_enabled: Indicates if errors messages should be sent.
""" """
bot_llm_enabled: bool = True bot_llm_enabled: bool = True
@@ -432,15 +676,18 @@ class RTVIObserver(BaseObserver):
Note: Note:
This observer only handles outgoing messages. Incoming RTVI client messages This observer only handles outgoing messages. Incoming RTVI client messages
are handled by the RTVIProcessor. are handled by the RTVIProcessor.
Args:
rtvi (RTVIProcessor): The RTVI processor to push frames to.
params (RTVIObserverParams): Settings to enable/disable specific messages.
""" """
def __init__( def __init__(
self, rtvi: "RTVIProcessor", *, params: Optional[RTVIObserverParams] = None, **kwargs self, rtvi: "RTVIProcessor", *, params: Optional[RTVIObserverParams] = None, **kwargs
): ):
"""Initialize the RTVI observer.
Args:
rtvi: The RTVI processor to push frames to.
params: Settings to enable/disable specific messages.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._rtvi = rtvi self._rtvi = rtvi
self._params = params or RTVIObserverParams() self._params = params or RTVIObserverParams()
@@ -452,11 +699,7 @@ class RTVIObserver(BaseObserver):
"""Process a frame being pushed through the pipeline. """Process a frame being pushed through the pipeline.
Args: Args:
src: Source processor pushing the frame data: Frame push event data containing source, frame, direction, and timestamp.
dst: Destination processor receiving the frame
frame: The frame being pushed
direction: Direction of frame flow in pipeline
timestamp: Time when frame was pushed
""" """
src = data.source src = data.source
frame = data.frame frame = data.frame
@@ -517,13 +760,14 @@ class RTVIObserver(BaseObserver):
"""Push an urgent transport message to the RTVI processor. """Push an urgent transport message to the RTVI processor.
Args: Args:
model: The message model to send model: The message model to send.
exclude_none: Whether to exclude None values from the model dump exclude_none: Whether to exclude None values from the model dump.
""" """
frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none))
await self._rtvi.push_frame(frame) await self._rtvi.push_frame(frame)
async def _push_bot_transcription(self): async def _push_bot_transcription(self):
"""Push accumulated bot transcription as a message."""
if len(self._bot_transcription) > 0: if len(self._bot_transcription) > 0:
message = RTVIBotTranscriptionMessage( message = RTVIBotTranscriptionMessage(
data=RTVITextMessageData(text=self._bot_transcription) data=RTVITextMessageData(text=self._bot_transcription)
@@ -532,6 +776,7 @@ class RTVIObserver(BaseObserver):
self._bot_transcription = "" self._bot_transcription = ""
async def _handle_interruptions(self, frame: Frame): async def _handle_interruptions(self, frame: Frame):
"""Handle user speaking interruption frames."""
message = None message = None
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, UserStartedSpeakingFrame):
message = RTVIUserStartedSpeakingMessage() message = RTVIUserStartedSpeakingMessage()
@@ -542,6 +787,7 @@ class RTVIObserver(BaseObserver):
await self.push_transport_message_urgent(message) await self.push_transport_message_urgent(message)
async def _handle_bot_speaking(self, frame: Frame): async def _handle_bot_speaking(self, frame: Frame):
"""Handle bot speaking event frames."""
message = None message = None
if isinstance(frame, BotStartedSpeakingFrame): if isinstance(frame, BotStartedSpeakingFrame):
message = RTVIBotStartedSpeakingMessage() message = RTVIBotStartedSpeakingMessage()
@@ -552,6 +798,7 @@ class RTVIObserver(BaseObserver):
await self.push_transport_message_urgent(message) await self.push_transport_message_urgent(message)
async def _handle_llm_text_frame(self, frame: LLMTextFrame): async def _handle_llm_text_frame(self, frame: LLMTextFrame):
"""Handle LLM text output frames."""
message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text))
await self.push_transport_message_urgent(message) await self.push_transport_message_urgent(message)
@@ -560,6 +807,7 @@ class RTVIObserver(BaseObserver):
await self._push_bot_transcription() await self._push_bot_transcription()
async def _handle_user_transcriptions(self, frame: Frame): async def _handle_user_transcriptions(self, frame: Frame):
"""Handle user transcription frames."""
message = None message = None
if isinstance(frame, TranscriptionFrame): if isinstance(frame, TranscriptionFrame):
message = RTVIUserTranscriptionMessage( message = RTVIUserTranscriptionMessage(
@@ -608,6 +856,7 @@ class RTVIObserver(BaseObserver):
logger.warning(f"Caught an error while trying to handle context: {e}") logger.warning(f"Caught an error while trying to handle context: {e}")
async def _handle_metrics(self, frame: MetricsFrame): async def _handle_metrics(self, frame: MetricsFrame):
"""Handle metrics frames and convert to RTVI metrics messages."""
metrics = {} metrics = {}
for d in frame.data: for d in frame.data:
if isinstance(d, TTFBMetricsData): if isinstance(d, TTFBMetricsData):
@@ -632,6 +881,13 @@ class RTVIObserver(BaseObserver):
class RTVIProcessor(FrameProcessor): class RTVIProcessor(FrameProcessor):
"""Main processor for handling RTVI protocol messages and actions.
This processor manages the RTVI protocol communication including client-server
handshaking, configuration management, action execution, and message routing.
It serves as the central hub for RTVI protocol operations.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -639,6 +895,13 @@ class RTVIProcessor(FrameProcessor):
transport: Optional[BaseTransport] = None, transport: Optional[BaseTransport] = None,
**kwargs, **kwargs,
): ):
"""Initialize the RTVI processor.
Args:
config: Initial RTVI configuration.
transport: Transport layer for communication.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._config = config or RTVIConfig(config=[]) self._config = config or RTVIConfig(config=[])
@@ -668,34 +931,67 @@ class RTVIProcessor(FrameProcessor):
self._input_transport.enable_audio_in_stream_on_start(False) self._input_transport.enable_audio_in_stream_on_start(False)
def register_action(self, action: RTVIAction): def register_action(self, action: RTVIAction):
"""Register an action that can be executed via RTVI.
Args:
action: The action to register.
"""
id = self._action_id(action.service, action.action) id = self._action_id(action.service, action.action)
self._registered_actions[id] = action self._registered_actions[id] = action
def register_service(self, service: RTVIService): def register_service(self, service: RTVIService):
"""Register a service that can be configured via RTVI.
Args:
service: The service to register.
"""
self._registered_services[service.name] = service self._registered_services[service.name] = service
async def set_client_ready(self): async def set_client_ready(self):
"""Mark the client as ready and trigger the ready event."""
self._client_ready = True self._client_ready = True
await self._call_event_handler("on_client_ready") await self._call_event_handler("on_client_ready")
async def set_bot_ready(self): async def set_bot_ready(self):
"""Mark the bot as ready and send the bot-ready message."""
self._bot_ready = True self._bot_ready = True
await self._update_config(self._config, False) await self._update_config(self._config, False)
await self._send_bot_ready() await self._send_bot_ready()
def set_errors_enabled(self, enabled: bool): def set_errors_enabled(self, enabled: bool):
"""Enable or disable error message sending.
Args:
enabled: Whether to send error messages.
"""
self._errors_enabled = enabled self._errors_enabled = enabled
async def interrupt_bot(self): async def interrupt_bot(self):
"""Send a bot interruption frame upstream."""
await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM)
async def send_error(self, error: str): async def send_error(self, error: str):
"""Send an error message to the client.
Args:
error: The error message to send.
"""
await self._send_error_frame(ErrorFrame(error=error)) await self._send_error_frame(ErrorFrame(error=error))
async def handle_message(self, message: RTVIMessage): async def handle_message(self, message: RTVIMessage):
"""Handle an incoming RTVI message.
Args:
message: The RTVI message to handle.
"""
await self._message_queue.put(message) await self._message_queue.put(message)
async def handle_function_call(self, params: FunctionCallParams): async def handle_function_call(self, params: FunctionCallParams):
"""Handle a function call from the LLM.
Args:
params: The function call parameters.
"""
fn = RTVILLMFunctionCallMessageData( fn = RTVILLMFunctionCallMessageData(
function_name=params.function_name, function_name=params.function_name,
tool_call_id=params.tool_call_id, tool_call_id=params.tool_call_id,
@@ -707,6 +1003,16 @@ class RTVIProcessor(FrameProcessor):
async def handle_function_call_start( async def handle_function_call_start(
self, function_name: str, llm: FrameProcessor, context: OpenAILLMContext self, function_name: str, llm: FrameProcessor, context: OpenAILLMContext
): ):
"""Handle the start of a function call from the LLM.
Args:
function_name: Name of the function being called.
llm: The LLM processor making the call.
context: The LLM context.
Note:
This method is deprecated. Use handle_function_call() instead.
"""
import warnings import warnings
with warnings.catch_warnings(): with warnings.catch_warnings():
@@ -721,6 +1027,12 @@ class RTVIProcessor(FrameProcessor):
await self._push_transport_message(message, exclude_none=False) await self._push_transport_message(message, exclude_none=False)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames through the RTVI processor.
Args:
frame: The frame to process.
direction: The direction of frame flow.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# Specific system frames # Specific system frames
@@ -754,6 +1066,7 @@ class RTVIProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def _start(self, frame: StartFrame): async def _start(self, frame: StartFrame):
"""Start the RTVI processor tasks."""
if not self._action_task: if not self._action_task:
self._action_queue = WatchdogQueue(self.task_manager) self._action_queue = WatchdogQueue(self.task_manager)
self._action_task = self.create_task(self._action_task_handler()) self._action_task = self.create_task(self._action_task_handler())
@@ -763,12 +1076,15 @@ class RTVIProcessor(FrameProcessor):
await self._call_event_handler("on_bot_started") await self._call_event_handler("on_bot_started")
async def _stop(self, frame: EndFrame): async def _stop(self, frame: EndFrame):
"""Stop the RTVI processor tasks."""
await self._cancel_tasks() await self._cancel_tasks()
async def _cancel(self, frame: CancelFrame): async def _cancel(self, frame: CancelFrame):
"""Cancel the RTVI processor tasks."""
await self._cancel_tasks() await self._cancel_tasks()
async def _cancel_tasks(self): async def _cancel_tasks(self):
"""Cancel all running tasks."""
if self._action_task: if self._action_task:
await self.cancel_task(self._action_task) await self.cancel_task(self._action_task)
self._action_task = None self._action_task = None
@@ -778,22 +1094,26 @@ class RTVIProcessor(FrameProcessor):
self._message_task = None self._message_task = None
async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True): async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True):
"""Push a transport message frame."""
frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none))
await self.push_frame(frame) await self.push_frame(frame)
async def _action_task_handler(self): async def _action_task_handler(self):
"""Handle incoming action frames."""
while True: while True:
frame = await self._action_queue.get() frame = await self._action_queue.get()
await self._handle_action(frame.message_id, frame.rtvi_action_run) await self._handle_action(frame.message_id, frame.rtvi_action_run)
self._action_queue.task_done() self._action_queue.task_done()
async def _message_task_handler(self): async def _message_task_handler(self):
"""Handle incoming transport messages."""
while True: while True:
message = await self._message_queue.get() message = await self._message_queue.get()
await self._handle_message(message) await self._handle_message(message)
self._message_queue.task_done() self._message_queue.task_done()
async def _handle_transport_message(self, frame: TransportMessageUrgentFrame): async def _handle_transport_message(self, frame: TransportMessageUrgentFrame):
"""Handle an incoming transport message frame."""
try: try:
transport_message = frame.message transport_message = frame.message
if transport_message.get("label") != RTVI_MESSAGE_LABEL: if transport_message.get("label") != RTVI_MESSAGE_LABEL:
@@ -806,6 +1126,7 @@ class RTVIProcessor(FrameProcessor):
logger.warning(f"Invalid RTVI transport message: {e}") logger.warning(f"Invalid RTVI transport message: {e}")
async def _handle_message(self, message: RTVIMessage): async def _handle_message(self, message: RTVIMessage):
"""Handle a parsed RTVI message."""
try: try:
match message.type: match message.type:
case "client-ready": case "client-ready":
@@ -842,6 +1163,7 @@ class RTVIProcessor(FrameProcessor):
logger.warning(f"Exception processing message: {e}") logger.warning(f"Exception processing message: {e}")
async def _handle_client_ready(self, request_id: str): async def _handle_client_ready(self, request_id: str):
"""Handle a client-ready message."""
logger.debug("Received client-ready") logger.debug("Received client-ready")
if self._input_transport: if self._input_transport:
await self._input_transport.start_audio_in_streaming() await self._input_transport.start_audio_in_streaming()
@@ -850,6 +1172,7 @@ class RTVIProcessor(FrameProcessor):
await self.set_client_ready() await self.set_client_ready()
async def _handle_audio_buffer(self, data): async def _handle_audio_buffer(self, data):
"""Handle incoming audio buffer data."""
if not self._input_transport: if not self._input_transport:
return return
@@ -871,20 +1194,24 @@ class RTVIProcessor(FrameProcessor):
logger.error(f"Error processing audio buffer: {e}") logger.error(f"Error processing audio buffer: {e}")
async def _handle_describe_config(self, request_id: str): async def _handle_describe_config(self, request_id: str):
"""Handle a describe-config request."""
services = list(self._registered_services.values()) services = list(self._registered_services.values())
message = RTVIDescribeConfig(id=request_id, data=RTVIDescribeConfigData(config=services)) message = RTVIDescribeConfig(id=request_id, data=RTVIDescribeConfigData(config=services))
await self._push_transport_message(message) await self._push_transport_message(message)
async def _handle_describe_actions(self, request_id: str): async def _handle_describe_actions(self, request_id: str):
"""Handle a describe-actions request."""
actions = list(self._registered_actions.values()) actions = list(self._registered_actions.values())
message = RTVIDescribeActions(id=request_id, data=RTVIDescribeActionsData(actions=actions)) message = RTVIDescribeActions(id=request_id, data=RTVIDescribeActionsData(actions=actions))
await self._push_transport_message(message) await self._push_transport_message(message)
async def _handle_get_config(self, request_id: str): async def _handle_get_config(self, request_id: str):
"""Handle a get-config request."""
message = RTVIConfigResponse(id=request_id, data=self._config) message = RTVIConfigResponse(id=request_id, data=self._config)
await self._push_transport_message(message) await self._push_transport_message(message)
def _update_config_option(self, service: str, config: RTVIServiceOptionConfig): def _update_config_option(self, service: str, config: RTVIServiceOptionConfig):
"""Update a specific configuration option."""
for service_config in self._config.config: for service_config in self._config.config:
if service_config.service == service: if service_config.service == service:
for option_config in service_config.options: for option_config in service_config.options:
@@ -896,6 +1223,7 @@ class RTVIProcessor(FrameProcessor):
service_config.options.append(config) service_config.options.append(config)
async def _update_service_config(self, config: RTVIServiceConfig): async def _update_service_config(self, config: RTVIServiceConfig):
"""Update configuration for a specific service."""
service = self._registered_services[config.service] service = self._registered_services[config.service]
for option in config.options: for option in config.options:
handler = service._options_dict[option.name].handler handler = service._options_dict[option.name].handler
@@ -903,16 +1231,19 @@ class RTVIProcessor(FrameProcessor):
self._update_config_option(service.name, option) self._update_config_option(service.name, option)
async def _update_config(self, data: RTVIConfig, interrupt: bool): async def _update_config(self, data: RTVIConfig, interrupt: bool):
"""Update the RTVI configuration."""
if interrupt: if interrupt:
await self.interrupt_bot() await self.interrupt_bot()
for service_config in data.config: for service_config in data.config:
await self._update_service_config(service_config) await self._update_service_config(service_config)
async def _handle_update_config(self, request_id: str, data: RTVIUpdateConfig): async def _handle_update_config(self, request_id: str, data: RTVIUpdateConfig):
"""Handle an update-config request."""
await self._update_config(RTVIConfig(config=data.config), data.interrupt) await self._update_config(RTVIConfig(config=data.config), data.interrupt)
await self._handle_get_config(request_id) await self._handle_get_config(request_id)
async def _handle_function_call_result(self, data): async def _handle_function_call_result(self, data):
"""Handle a function call result from the client."""
frame = FunctionCallResultFrame( frame = FunctionCallResultFrame(
function_name=data.function_name, function_name=data.function_name,
tool_call_id=data.tool_call_id, tool_call_id=data.tool_call_id,
@@ -922,6 +1253,7 @@ class RTVIProcessor(FrameProcessor):
await self.push_frame(frame) await self.push_frame(frame)
async def _handle_action(self, request_id: Optional[str], data: RTVIActionRun): async def _handle_action(self, request_id: Optional[str], data: RTVIActionRun):
"""Handle an action execution request."""
action_id = self._action_id(data.service, data.action) action_id = self._action_id(data.service, data.action)
if action_id not in self._registered_actions: if action_id not in self._registered_actions:
await self._send_error_response(request_id, f"Action {action_id} not registered") await self._send_error_response(request_id, f"Action {action_id} not registered")
@@ -939,6 +1271,7 @@ class RTVIProcessor(FrameProcessor):
await self._push_transport_message(message) await self._push_transport_message(message)
async def _send_bot_ready(self): async def _send_bot_ready(self):
"""Send the bot-ready message to the client."""
message = RTVIBotReady( message = RTVIBotReady(
id=self._client_ready_id, id=self._client_ready_id,
data=RTVIBotReadyData(version=RTVI_PROTOCOL_VERSION, config=self._config.config), data=RTVIBotReadyData(version=RTVI_PROTOCOL_VERSION, config=self._config.config),
@@ -946,14 +1279,17 @@ class RTVIProcessor(FrameProcessor):
await self._push_transport_message(message) await self._push_transport_message(message)
async def _send_error_frame(self, frame: ErrorFrame): async def _send_error_frame(self, frame: ErrorFrame):
"""Send an error frame as an RTVI error message."""
if self._errors_enabled: if self._errors_enabled:
message = RTVIError(data=RTVIErrorData(error=frame.error, fatal=frame.fatal)) message = RTVIError(data=RTVIErrorData(error=frame.error, fatal=frame.fatal))
await self._push_transport_message(message) await self._push_transport_message(message)
async def _send_error_response(self, id: str, error: str): async def _send_error_response(self, id: str, error: str):
"""Send an error response message."""
if self._errors_enabled: if self._errors_enabled:
message = RTVIErrorResponse(id=id, data=RTVIErrorResponseData(error=error)) message = RTVIErrorResponse(id=id, data=RTVIErrorResponseData(error=error))
await self._push_transport_message(message) await self._push_transport_message(message)
def _action_id(self, service: str, action: str) -> str: def _action_id(self, service: str, action: str) -> str:
"""Generate an action ID from service and action names."""
return f"{service}:{action}" return f"{service}:{action}"

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""GStreamer pipeline source integration for Pipecat."""
import asyncio import asyncio
from typing import Optional from typing import Optional
@@ -36,7 +38,24 @@ except ModuleNotFoundError as e:
class GStreamerPipelineSource(FrameProcessor): class GStreamerPipelineSource(FrameProcessor):
"""A frame processor that uses GStreamer pipelines as media sources.
This processor creates and manages GStreamer pipelines to generate audio and video
output frames. It handles pipeline lifecycle, decoding, format conversion, and
frame generation with configurable output parameters.
"""
class OutputParams(BaseModel): class OutputParams(BaseModel):
"""Output configuration parameters for GStreamer pipeline.
Parameters:
video_width: Width of output video frames in pixels.
video_height: Height of output video frames in pixels.
audio_sample_rate: Sample rate for audio output. If None, uses frame sample rate.
audio_channels: Number of audio channels for output.
clock_sync: Whether to synchronize output with pipeline clock.
"""
video_width: int = 1280 video_width: int = 1280
video_height: int = 720 video_height: int = 720
audio_sample_rate: Optional[int] = None audio_sample_rate: Optional[int] = None
@@ -44,6 +63,13 @@ class GStreamerPipelineSource(FrameProcessor):
clock_sync: bool = True clock_sync: bool = True
def __init__(self, *, pipeline: str, out_params: Optional[OutputParams] = None, **kwargs): def __init__(self, *, pipeline: str, out_params: Optional[OutputParams] = None, **kwargs):
"""Initialize the GStreamer pipeline source.
Args:
pipeline: GStreamer pipeline description string for the source.
out_params: Output configuration parameters. If None, uses defaults.
**kwargs: Additional arguments passed to parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._out_params = out_params or GStreamerPipelineSource.OutputParams() self._out_params = out_params or GStreamerPipelineSource.OutputParams()
@@ -67,6 +93,12 @@ class GStreamerPipelineSource(FrameProcessor):
bus.connect("message", self._on_gstreamer_message) bus.connect("message", self._on_gstreamer_message)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and manage GStreamer pipeline lifecycle.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# Specific system frames # Specific system frames
@@ -92,13 +124,16 @@ class GStreamerPipelineSource(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def _start(self, frame: StartFrame): async def _start(self, frame: StartFrame):
"""Start the GStreamer pipeline."""
self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate
self._player.set_state(Gst.State.PLAYING) self._player.set_state(Gst.State.PLAYING)
async def _stop(self, frame: EndFrame): async def _stop(self, frame: EndFrame):
"""Stop the GStreamer pipeline."""
self._player.set_state(Gst.State.NULL) self._player.set_state(Gst.State.NULL)
async def _cancel(self, frame: CancelFrame): async def _cancel(self, frame: CancelFrame):
"""Cancel the GStreamer pipeline."""
self._player.set_state(Gst.State.NULL) self._player.set_state(Gst.State.NULL)
# #
@@ -106,6 +141,7 @@ class GStreamerPipelineSource(FrameProcessor):
# #
def _on_gstreamer_message(self, bus: Gst.Bus, message: Gst.Message): def _on_gstreamer_message(self, bus: Gst.Bus, message: Gst.Message):
"""Handle GStreamer bus messages."""
t = message.type t = message.type
if t == Gst.MessageType.ERROR: if t == Gst.MessageType.ERROR:
err, debug = message.parse_error() err, debug = message.parse_error()
@@ -113,6 +149,7 @@ class GStreamerPipelineSource(FrameProcessor):
return True return True
def _decodebin_callback(self, decodebin: Gst.Element, pad: Gst.Pad): def _decodebin_callback(self, decodebin: Gst.Element, pad: Gst.Pad):
"""Handle new pads from decodebin element."""
caps_string = pad.get_current_caps().to_string() caps_string = pad.get_current_caps().to_string()
if caps_string.startswith("audio"): if caps_string.startswith("audio"):
self._decodebin_audio(pad) self._decodebin_audio(pad)
@@ -120,6 +157,7 @@ class GStreamerPipelineSource(FrameProcessor):
self._decodebin_video(pad) self._decodebin_video(pad)
def _decodebin_audio(self, pad: Gst.Pad): def _decodebin_audio(self, pad: Gst.Pad):
"""Set up audio processing pipeline from decoded audio pad."""
queue_audio = Gst.ElementFactory.make("queue", None) queue_audio = Gst.ElementFactory.make("queue", None)
audioconvert = Gst.ElementFactory.make("audioconvert", None) audioconvert = Gst.ElementFactory.make("audioconvert", None)
audioresample = Gst.ElementFactory.make("audioresample", None) audioresample = Gst.ElementFactory.make("audioresample", None)
@@ -153,6 +191,7 @@ class GStreamerPipelineSource(FrameProcessor):
pad.link(queue_pad) pad.link(queue_pad)
def _decodebin_video(self, pad: Gst.Pad): def _decodebin_video(self, pad: Gst.Pad):
"""Set up video processing pipeline from decoded video pad."""
queue_video = Gst.ElementFactory.make("queue", None) queue_video = Gst.ElementFactory.make("queue", None)
videoconvert = Gst.ElementFactory.make("videoconvert", None) videoconvert = Gst.ElementFactory.make("videoconvert", None)
videoscale = Gst.ElementFactory.make("videoscale", None) videoscale = Gst.ElementFactory.make("videoscale", None)
@@ -187,6 +226,7 @@ class GStreamerPipelineSource(FrameProcessor):
pad.link(queue_pad) pad.link(queue_pad)
def _appsink_audio_new_sample(self, appsink: GstApp.AppSink): def _appsink_audio_new_sample(self, appsink: GstApp.AppSink):
"""Handle new audio samples from GStreamer appsink."""
buffer = appsink.pull_sample().get_buffer() buffer = appsink.pull_sample().get_buffer()
(_, info) = buffer.map(Gst.MapFlags.READ) (_, info) = buffer.map(Gst.MapFlags.READ)
frame = OutputAudioRawFrame( frame = OutputAudioRawFrame(
@@ -199,6 +239,7 @@ class GStreamerPipelineSource(FrameProcessor):
return Gst.FlowReturn.OK return Gst.FlowReturn.OK
def _appsink_video_new_sample(self, appsink: GstApp.AppSink): def _appsink_video_new_sample(self, appsink: GstApp.AppSink):
"""Handle new video samples from GStreamer appsink."""
buffer = appsink.pull_sample().get_buffer() buffer = appsink.pull_sample().get_buffer()
(_, info) = buffer.map(Gst.MapFlags.READ) (_, info) = buffer.map(Gst.MapFlags.READ)
frame = OutputImageRawFrame( frame = OutputImageRawFrame(

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Idle frame processor for timeout-based callback execution."""
import asyncio import asyncio
from typing import Awaitable, Callable, List, Optional from typing import Awaitable, Callable, List, Optional
@@ -12,9 +14,11 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class IdleFrameProcessor(FrameProcessor): class IdleFrameProcessor(FrameProcessor):
"""This class waits to receive any frame or list of desired frames within a """Monitors frame activity and triggers callbacks on timeout.
given timeout. If the timeout is reached before receiving any of those
frames the provided callback will be called. This processor waits to receive any frame or specific frame types within a
given timeout period. If the timeout is reached before receiving the expected
frames, the provided callback will be executed.
""" """
def __init__( def __init__(
@@ -25,6 +29,16 @@ class IdleFrameProcessor(FrameProcessor):
types: Optional[List[type]] = None, types: Optional[List[type]] = None,
**kwargs, **kwargs,
): ):
"""Initialize the idle frame processor.
Args:
callback: Async callback function to execute on timeout. Receives
this processor instance as an argument.
timeout: Timeout duration in seconds before triggering the callback.
types: Optional list of frame types to monitor. If None, monitors
all frames.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._callback = callback self._callback = callback
@@ -33,6 +47,12 @@ class IdleFrameProcessor(FrameProcessor):
self._idle_task = None self._idle_task = None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and manage idle timeout monitoring.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
@@ -50,15 +70,18 @@ class IdleFrameProcessor(FrameProcessor):
self._idle_event.set() self._idle_event.set()
async def cleanup(self): async def cleanup(self):
"""Clean up resources and cancel pending tasks."""
if self._idle_task: if self._idle_task:
await self.cancel_task(self._idle_task) await self.cancel_task(self._idle_task)
def _create_idle_task(self): def _create_idle_task(self):
"""Create and start the idle monitoring task."""
if not self._idle_task: if not self._idle_task:
self._idle_event = asyncio.Event() self._idle_event = asyncio.Event()
self._idle_task = self.create_task(self._idle_task_handler()) self._idle_task = self.create_task(self._idle_task_handler())
async def _idle_task_handler(self): async def _idle_task_handler(self):
"""Handle idle timeout monitoring and callback execution."""
while True: while True:
try: try:
await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout) await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout)

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Frame logging utilities for debugging and monitoring frame flow in Pipecat pipelines."""
from typing import Optional, Tuple, Type from typing import Optional, Tuple, Type
from loguru import logger from loguru import logger
@@ -21,6 +23,13 @@ logger = logger.opt(ansi=True)
class FrameLogger(FrameProcessor): class FrameLogger(FrameProcessor):
"""A frame processor that logs frame information for debugging purposes.
This processor intercepts frames passing through the pipeline and logs
their details with configurable formatting and filtering. Useful for
debugging frame flow and understanding pipeline behavior.
"""
def __init__( def __init__(
self, self,
prefix="Frame", prefix="Frame",
@@ -32,12 +41,26 @@ class FrameLogger(FrameProcessor):
TransportMessageFrame, TransportMessageFrame,
), ),
): ):
"""Initialize the frame logger.
Args:
prefix: Text prefix to add to log messages. Defaults to "Frame".
color: ANSI color code for log message formatting. If None, no coloring is applied.
ignored_frame_types: Tuple of frame types to exclude from logging.
Defaults to common high-frequency frames like audio and speaking frames.
"""
super().__init__() super().__init__()
self._prefix = prefix self._prefix = prefix
self._color = color self._color = color
self._ignored_frame_types = ignored_frame_types self._ignored_frame_types = ignored_frame_types
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process and log frame information.
Args:
frame: The frame to process and potentially log.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if self._ignored_frame_types and not isinstance(frame, self._ignored_frame_types): if self._ignored_frame_types and not isinstance(frame, self._ignored_frame_types):

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Frame processor metrics collection and reporting."""
import time import time
from typing import Optional from typing import Optional
@@ -23,7 +25,20 @@ from pipecat.utils.base_object import BaseObject
class FrameProcessorMetrics(BaseObject): class FrameProcessorMetrics(BaseObject):
"""Metrics collection and reporting for frame processors.
Provides comprehensive metrics tracking for frame processing operations,
including timing measurements, resource usage, and performance analytics.
Supports TTFB tracking, processing duration metrics, and usage statistics
for LLM and TTS operations.
"""
def __init__(self): def __init__(self):
"""Initialize the frame processor metrics collector.
Sets up internal state for tracking various metrics including TTFB,
processing times, and usage statistics.
"""
super().__init__() super().__init__()
self._task_manager = None self._task_manager = None
self._start_ttfb_time = 0 self._start_ttfb_time = 0
@@ -32,13 +47,24 @@ class FrameProcessorMetrics(BaseObject):
self._should_report_ttfb = True self._should_report_ttfb = True
async def setup(self, task_manager: BaseTaskManager): async def setup(self, task_manager: BaseTaskManager):
"""Set up the metrics collector with a task manager.
Args:
task_manager: The task manager for handling async operations.
"""
self._task_manager = task_manager self._task_manager = task_manager
async def cleanup(self): async def cleanup(self):
"""Clean up metrics collection resources."""
await super().cleanup() await super().cleanup()
@property @property
def task_manager(self) -> BaseTaskManager: def task_manager(self) -> BaseTaskManager:
"""Get the associated task manager.
Returns:
The task manager instance for async operations.
"""
return self._task_manager return self._task_manager
@property @property
@@ -46,7 +72,7 @@ class FrameProcessorMetrics(BaseObject):
"""Get the current TTFB value in seconds. """Get the current TTFB value in seconds.
Returns: Returns:
Optional[float]: The TTFB value in seconds, or None if not measured The TTFB value in seconds, or None if not measured.
""" """
if self._last_ttfb_time > 0: if self._last_ttfb_time > 0:
return self._last_ttfb_time return self._last_ttfb_time
@@ -58,24 +84,46 @@ class FrameProcessorMetrics(BaseObject):
return None return None
def _processor_name(self): def _processor_name(self):
"""Get the processor name from core metrics data."""
return self._core_metrics_data.processor return self._core_metrics_data.processor
def _model_name(self): def _model_name(self):
"""Get the model name from core metrics data."""
return self._core_metrics_data.model return self._core_metrics_data.model
def set_core_metrics_data(self, data: MetricsData): def set_core_metrics_data(self, data: MetricsData):
"""Set the core metrics data for this collector.
Args:
data: The core metrics data containing processor and model information.
"""
self._core_metrics_data = data self._core_metrics_data = data
def set_processor_name(self, name: str): def set_processor_name(self, name: str):
"""Set the processor name for metrics reporting.
Args:
name: The name of the processor to use in metrics.
"""
self._core_metrics_data = MetricsData(processor=name) self._core_metrics_data = MetricsData(processor=name)
async def start_ttfb_metrics(self, report_only_initial_ttfb): async def start_ttfb_metrics(self, report_only_initial_ttfb):
"""Start measuring time-to-first-byte (TTFB).
Args:
report_only_initial_ttfb: Whether to report only the first TTFB measurement.
"""
if self._should_report_ttfb: if self._should_report_ttfb:
self._start_ttfb_time = time.time() self._start_ttfb_time = time.time()
self._last_ttfb_time = 0 self._last_ttfb_time = 0
self._should_report_ttfb = not report_only_initial_ttfb self._should_report_ttfb = not report_only_initial_ttfb
async def stop_ttfb_metrics(self): async def stop_ttfb_metrics(self):
"""Stop TTFB measurement and generate metrics frame.
Returns:
MetricsFrame containing TTFB data, or None if not measuring.
"""
if self._start_ttfb_time == 0: if self._start_ttfb_time == 0:
return None return None
@@ -88,9 +136,15 @@ class FrameProcessorMetrics(BaseObject):
return MetricsFrame(data=[ttfb]) return MetricsFrame(data=[ttfb])
async def start_processing_metrics(self): async def start_processing_metrics(self):
"""Start measuring processing time."""
self._start_processing_time = time.time() self._start_processing_time = time.time()
async def stop_processing_metrics(self): async def stop_processing_metrics(self):
"""Stop processing time measurement and generate metrics frame.
Returns:
MetricsFrame containing processing duration data, or None if not measuring.
"""
if self._start_processing_time == 0: if self._start_processing_time == 0:
return None return None
@@ -103,6 +157,14 @@ class FrameProcessorMetrics(BaseObject):
return MetricsFrame(data=[processing]) return MetricsFrame(data=[processing])
async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): async def start_llm_usage_metrics(self, tokens: LLMTokenUsage):
"""Record LLM token usage metrics.
Args:
tokens: Token usage information including prompt and completion tokens.
Returns:
MetricsFrame containing LLM usage data.
"""
logger.debug( logger.debug(
f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}" f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}"
) )
@@ -112,6 +174,14 @@ class FrameProcessorMetrics(BaseObject):
return MetricsFrame(data=[value]) return MetricsFrame(data=[value])
async def start_tts_usage_metrics(self, text: str): async def start_tts_usage_metrics(self, text: str):
"""Record TTS character usage metrics.
Args:
text: The text being processed by TTS.
Returns:
MetricsFrame containing TTS usage data.
"""
characters = TTSUsageMetricsData( characters = TTSUsageMetricsData(
processor=self._processor_name(), model=self._model_name(), value=len(text) processor=self._processor_name(), model=self._model_name(), value=len(text)
) )

View File

@@ -4,7 +4,7 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio """Sentry integration for frame processor metrics."""
from loguru import logger from loguru import logger
@@ -22,7 +22,19 @@ from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMet
class SentryMetrics(FrameProcessorMetrics): class SentryMetrics(FrameProcessorMetrics):
"""Frame processor metrics integration with Sentry monitoring.
Extends FrameProcessorMetrics to send time-to-first-byte (TTFB) and
processing metrics as Sentry transactions for performance monitoring
and debugging.
"""
def __init__(self): def __init__(self):
"""Initialize the Sentry metrics collector.
Sets up internal state for tracking transactions and verifies
Sentry SDK initialization status.
"""
super().__init__() super().__init__()
self._ttfb_metrics_tx = None self._ttfb_metrics_tx = None
self._processing_metrics_tx = None self._processing_metrics_tx = None
@@ -32,6 +44,11 @@ class SentryMetrics(FrameProcessorMetrics):
self._sentry_task = None self._sentry_task = None
async def setup(self, task_manager: BaseTaskManager): async def setup(self, task_manager: BaseTaskManager):
"""Setup the Sentry metrics system.
Args:
task_manager: The task manager to use for background operations.
"""
await super().setup(task_manager) await super().setup(task_manager)
if self._sentry_available: if self._sentry_available:
self._sentry_queue = WatchdogQueue(task_manager) self._sentry_queue = WatchdogQueue(task_manager)
@@ -40,6 +57,10 @@ class SentryMetrics(FrameProcessorMetrics):
) )
async def cleanup(self): async def cleanup(self):
"""Clean up Sentry resources and flush pending transactions.
Ensures all pending transactions are sent to Sentry before shutdown.
"""
await super().cleanup() await super().cleanup()
if self._sentry_task: if self._sentry_task:
await self._sentry_queue.put(None) await self._sentry_queue.put(None)
@@ -49,6 +70,11 @@ class SentryMetrics(FrameProcessorMetrics):
sentry_sdk.flush(timeout=5.0) sentry_sdk.flush(timeout=5.0)
async def start_ttfb_metrics(self, report_only_initial_ttfb): async def start_ttfb_metrics(self, report_only_initial_ttfb):
"""Start tracking time-to-first-byte metrics.
Args:
report_only_initial_ttfb: Whether to report only the initial TTFB measurement.
"""
await super().start_ttfb_metrics(report_only_initial_ttfb) await super().start_ttfb_metrics(report_only_initial_ttfb)
if self._should_report_ttfb and self._sentry_available: if self._should_report_ttfb and self._sentry_available:
@@ -61,6 +87,10 @@ class SentryMetrics(FrameProcessorMetrics):
) )
async def stop_ttfb_metrics(self): async def stop_ttfb_metrics(self):
"""Stop tracking time-to-first-byte metrics.
Queues the TTFB transaction for completion and transmission to Sentry.
"""
await super().stop_ttfb_metrics() await super().stop_ttfb_metrics()
if self._sentry_available and self._ttfb_metrics_tx: if self._sentry_available and self._ttfb_metrics_tx:
@@ -68,6 +98,10 @@ class SentryMetrics(FrameProcessorMetrics):
self._ttfb_metrics_tx = None self._ttfb_metrics_tx = None
async def start_processing_metrics(self): async def start_processing_metrics(self):
"""Start tracking frame processing metrics.
Creates a new Sentry transaction to track processing performance.
"""
await super().start_processing_metrics() await super().start_processing_metrics()
if self._sentry_available: if self._sentry_available:
@@ -80,6 +114,10 @@ class SentryMetrics(FrameProcessorMetrics):
) )
async def stop_processing_metrics(self): async def stop_processing_metrics(self):
"""Stop tracking frame processing metrics.
Queues the processing transaction for completion and transmission to Sentry.
"""
await super().stop_processing_metrics() await super().stop_processing_metrics()
if self._sentry_available and self._processing_metrics_tx: if self._sentry_available and self._processing_metrics_tx:
@@ -87,6 +125,7 @@ class SentryMetrics(FrameProcessorMetrics):
self._processing_metrics_tx = None self._processing_metrics_tx = None
async def _sentry_task_handler(self): async def _sentry_task_handler(self):
"""Background task handler for completing Sentry transactions."""
running = True running = True
while running: while running:
tx = await self._sentry_queue.get() tx = await self._sentry_queue.get()

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Producer processor for frame filtering and distribution."""
import asyncio import asyncio
from typing import Awaitable, Callable, List from typing import Awaitable, Callable, List
@@ -13,15 +15,24 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
async def identity_transformer(frame: Frame): async def identity_transformer(frame: Frame):
"""Default transformer that returns the frame unchanged.
Args:
frame: The frame to transform.
Returns:
The same frame without modifications.
"""
return frame return frame
class ProducerProcessor(FrameProcessor): class ProducerProcessor(FrameProcessor):
"""This class optionally passes-through received frames and decides if those """A processor that filters frames and distributes them to multiple consumers.
frames should be sent to consumers based on a user-defined filter. The
frames can be transformed into a different type of frame before being
sending them to the consumers. More than one consumer can be added.
This processor receives frames, applies a filter to determine which frames
should be sent to consumers (ConsumerProcessor), optionally transforms those
frames, and distributes them to registered consumer queues. It can also pass
frames through to the next processor in the pipeline.
""" """
def __init__( def __init__(
@@ -31,6 +42,16 @@ class ProducerProcessor(FrameProcessor):
transformer: Callable[[Frame], Awaitable[Frame]] = identity_transformer, transformer: Callable[[Frame], Awaitable[Frame]] = identity_transformer,
passthrough: bool = True, passthrough: bool = True,
): ):
"""Initialize the producer processor.
Args:
filter: Async function that determines if a frame should be produced.
Must return True for frames to be sent to consumers.
transformer: Async function to transform frames before sending to consumers.
Defaults to identity_transformer which returns frames unchanged.
passthrough: Whether to pass frames through to the next processor.
If True, all frames continue downstream regardless of filter result.
"""
super().__init__() super().__init__()
self._filter = filter self._filter = filter
self._transformer = transformer self._transformer = transformer
@@ -38,8 +59,7 @@ class ProducerProcessor(FrameProcessor):
self._consumers: List[asyncio.Queue] = [] self._consumers: List[asyncio.Queue] = []
def add_consumer(self): def add_consumer(self):
""" """Add a new consumer and return its associated queue.
Adds a new consumer and returns its associated queue.
Returns: Returns:
asyncio.Queue: The queue for the newly added consumer. asyncio.Queue: The queue for the newly added consumer.
@@ -49,15 +69,15 @@ class ProducerProcessor(FrameProcessor):
return queue return queue
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
""" """Process an incoming frame and determine whether to produce it.
Processes an incoming frame and determines whether to produce it as a ProducerItem.
If the frame meets the produce criteria, it will be added to the consumer queues. If the frame meets the filter criteria, it will be transformed and added
If passthrough is enabled, the frame will also be sent to consumers. to all consumer queues. If passthrough is enabled, the original frame
will also be sent downstream.
Args: Args:
frame (Frame): The frame to process. frame: The frame to process.
direction (FrameDirection): The direction of the frame. direction: The direction of the frame flow.
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
@@ -69,6 +89,7 @@ class ProducerProcessor(FrameProcessor):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def _produce(self, frame: Frame): async def _produce(self, frame: Frame):
"""Produce a frame to all consumers."""
for consumer in self._consumers: for consumer in self._consumers:
new_frame = await self._transformer(frame) new_frame = await self._transformer(frame)
await consumer.put(new_frame) await consumer.put(new_frame)

View File

@@ -4,14 +4,20 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
from typing import Coroutine """Stateless text transformation processor for Pipecat."""
from typing import Callable, Coroutine, Union
from pipecat.frames.frames import Frame, TextFrame from pipecat.frames.frames import Frame, TextFrame
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class StatelessTextTransformer(FrameProcessor): class StatelessTextTransformer(FrameProcessor):
"""This processor calls the given function on any text in a text frame. """Processor that applies transformation functions to text frames.
This processor intercepts TextFrame objects and applies a user-provided
transformation function to the text content. The function can be either
synchronous or asynchronous (coroutine).
>>> async def print_frames(aggregator, frame): >>> async def print_frames(aggregator, frame):
... async for frame in aggregator.process_frame(frame): ... async for frame in aggregator.process_frame(frame):
@@ -22,11 +28,25 @@ class StatelessTextTransformer(FrameProcessor):
HELLO HELLO
""" """
def __init__(self, transform_fn): def __init__(
self, transform_fn: Union[Callable[[str], str], Callable[[str], Coroutine[None, None, str]]]
):
"""Initialize the text transformer.
Args:
transform_fn: Function to apply to text content. Can be synchronous
(str -> str) or asynchronous (str -> Coroutine[None, None, str]).
"""
super().__init__() super().__init__()
self._transform_fn = transform_fn self._transform_fn = transform_fn
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames, applying transformation to text frames.
Args:
frame: The frame to process.
direction: The direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Transcript processing utilities for conversation recording and analysis.
This module provides processors that convert speech and text frames into structured
transcript messages with timestamps, enabling conversation history tracking and analysis.
"""
from typing import List, Optional from typing import List, Optional
from loguru import logger from loguru import logger
@@ -30,7 +36,11 @@ class BaseTranscriptProcessor(FrameProcessor):
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize processor with empty message store.""" """Initialize processor with empty message store.
Args:
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._processed_messages: List[TranscriptionMessage] = [] self._processed_messages: List[TranscriptionMessage] = []
self._register_event_handler("on_transcript_update") self._register_event_handler("on_transcript_update")
@@ -39,7 +49,7 @@ class BaseTranscriptProcessor(FrameProcessor):
"""Emit transcript updates for new messages. """Emit transcript updates for new messages.
Args: Args:
messages: New messages to emit in update messages: New messages to emit in update.
""" """
if messages: if messages:
self._processed_messages.extend(messages) self._processed_messages.extend(messages)
@@ -55,8 +65,8 @@ class UserTranscriptProcessor(BaseTranscriptProcessor):
"""Process TranscriptionFrames into user conversation messages. """Process TranscriptionFrames into user conversation messages.
Args: Args:
frame: Input frame to process frame: Input frame to process.
direction: Frame processing direction direction: Frame processing direction.
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
@@ -77,14 +87,14 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
- The bot stops speaking (BotStoppedSpeakingFrame) - The bot stops speaking (BotStoppedSpeakingFrame)
- The bot is interrupted (StartInterruptionFrame) - The bot is interrupted (StartInterruptionFrame)
- The pipeline ends (EndFrame) - The pipeline ends (EndFrame)
Attributes:
_current_text_parts: List of text fragments being aggregated for current utterance
_aggregation_start_time: Timestamp when the current utterance began
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize processor with aggregation state.""" """Initialize processor with aggregation state.
Args:
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._current_text_parts: List[str] = [] self._current_text_parts: List[str] = []
self._aggregation_start_time: Optional[str] = None self._aggregation_start_time: Optional[str] = None
@@ -176,8 +186,8 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
- CancelFrame: Completes current utterance due to cancellation - CancelFrame: Completes current utterance due to cancellation
Args: Args:
frame: Input frame to process frame: Input frame to process.
direction: Frame processing direction direction: Frame processing direction.
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
@@ -245,7 +255,10 @@ class TranscriptProcessor:
"""Get the user transcript processor. """Get the user transcript processor.
Args: Args:
**kwargs: Arguments specific to UserTranscriptProcessor **kwargs: Arguments specific to UserTranscriptProcessor.
Returns:
The user transcript processor instance.
""" """
if self._user_processor is None: if self._user_processor is None:
self._user_processor = UserTranscriptProcessor(**kwargs) self._user_processor = UserTranscriptProcessor(**kwargs)
@@ -262,7 +275,10 @@ class TranscriptProcessor:
"""Get the assistant transcript processor. """Get the assistant transcript processor.
Args: Args:
**kwargs: Arguments specific to AssistantTranscriptProcessor **kwargs: Arguments specific to AssistantTranscriptProcessor.
Returns:
The assistant transcript processor instance.
""" """
if self._assistant_processor is None: if self._assistant_processor is None:
self._assistant_processor = AssistantTranscriptProcessor(**kwargs) self._assistant_processor = AssistantTranscriptProcessor(**kwargs)
@@ -279,10 +295,10 @@ class TranscriptProcessor:
"""Register event handler for both processors. """Register event handler for both processors.
Args: Args:
event_name: Name of event to handle event_name: Name of event to handle.
Returns: Returns:
Decorator function that registers handler with both processors Decorator function that registers handler with both processors.
""" """
def decorator(handler): def decorator(handler):

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""User idle detection and timeout handling for Pipecat."""
import asyncio import asyncio
import inspect import inspect
from typing import Awaitable, Callable, Union from typing import Awaitable, Callable, Union
@@ -22,19 +24,12 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class UserIdleProcessor(FrameProcessor): class UserIdleProcessor(FrameProcessor):
"""Monitors user inactivity and triggers callbacks after timeout periods. """Monitors user inactivity and triggers callbacks after timeout periods.
Starts monitoring only after the first conversation activity (UserStartedSpeaking This processor tracks user activity and triggers configurable callbacks when
or BotSpeaking). users become idle. It starts monitoring only after the first conversation
activity and supports both basic and retry-based callback patterns.
Args:
callback: Function to call when user is idle. Can be either:
- Basic callback(processor) -> None
- Retry callback(processor, retry_count) -> bool
Return True to continue monitoring for idle events,
Return False to stop the idle monitoring task
timeout: Seconds to wait before considering user idle
**kwargs: Additional arguments passed to FrameProcessor
Example: Example:
```
# Retry callback: # Retry callback:
async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool: async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool:
if retry_count < 3: if retry_count < 3:
@@ -50,6 +45,7 @@ class UserIdleProcessor(FrameProcessor):
callback=handle_idle, callback=handle_idle,
timeout=5.0 timeout=5.0
) )
```
""" """
def __init__( def __init__(
@@ -62,6 +58,17 @@ class UserIdleProcessor(FrameProcessor):
timeout: float, timeout: float,
**kwargs, **kwargs,
): ):
"""Initialize the user idle processor.
Args:
callback: Function to call when user is idle. Can be either:
- Basic callback(processor) -> None
- Retry callback(processor, retry_count) -> bool
Return True to continue monitoring for idle events,
Return False to stop the idle monitoring task
timeout: Seconds to wait before considering user idle.
**kwargs: Additional arguments passed to FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._callback = self._wrap_callback(callback) self._callback = self._wrap_callback(callback)
self._timeout = timeout self._timeout = timeout
@@ -107,7 +114,11 @@ class UserIdleProcessor(FrameProcessor):
@property @property
def retry_count(self) -> int: def retry_count(self) -> int:
"""Returns the current retry count.""" """Get the current retry count.
Returns:
The number of times the idle callback has been triggered.
"""
return self._retry_count return self._retry_count
async def _stop(self) -> None: async def _stop(self) -> None:
@@ -120,8 +131,8 @@ class UserIdleProcessor(FrameProcessor):
"""Processes incoming frames and manages idle monitoring state. """Processes incoming frames and manages idle monitoring state.
Args: Args:
frame: The frame to process frame: The frame to process.
direction: Direction of the frame flow direction: Direction of the frame flow.
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)

View File

@@ -32,12 +32,14 @@ class AIService(FrameProcessor):
settings handling, session properties, and frame processing lifecycle. settings handling, session properties, and frame processing lifecycle.
Subclasses should implement specific AI functionality while leveraging Subclasses should implement specific AI functionality while leveraging
this base infrastructure. this base infrastructure.
Args:
**kwargs: Additional arguments passed to the parent FrameProcessor.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the AI service.
Args:
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._model_name: str = "" self._model_name: str = ""
self._settings: Dict[str, Any] = {} self._settings: Dict[str, Any] = {}

View File

@@ -101,13 +101,6 @@ class AnthropicLLMService(LLMService):
Provides inference capabilities with Claude models including support for Provides inference capabilities with Claude models including support for
function calling, vision processing, streaming responses, and prompt caching. function calling, vision processing, streaming responses, and prompt caching.
Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex. Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex.
Args:
api_key: Anthropic API key for authentication.
model: Model name to use. Defaults to "claude-sonnet-4-20250514".
params: Optional model parameters for inference.
client: Optional custom Anthropic client instance.
**kwargs: Additional arguments passed to parent LLMService.
""" """
# Overriding the default adapter to use the Anthropic one. # Overriding the default adapter to use the Anthropic one.
@@ -141,6 +134,15 @@ class AnthropicLLMService(LLMService):
client=None, client=None,
**kwargs, **kwargs,
): ):
"""Initialize the Anthropic LLM service.
Args:
api_key: Anthropic API key for authentication.
model: Model name to use. Defaults to "claude-sonnet-4-20250514".
params: Optional model parameters for inference.
client: Optional custom Anthropic client instance.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
params = params or AnthropicLLMService.InputParams() params = params or AnthropicLLMService.InputParams()
self._client = client or AsyncAnthropic( self._client = client or AsyncAnthropic(
@@ -425,12 +427,6 @@ class AnthropicLLMContext(OpenAILLMContext):
Extends OpenAILLMContext to handle Anthropic-specific features like Extends OpenAILLMContext to handle Anthropic-specific features like
system messages, prompt caching, and message format conversions. system messages, prompt caching, and message format conversions.
Manages conversation state and message history formatting. Manages conversation state and message history formatting.
Args:
messages: Initial list of conversation messages.
tools: Available function calling tools.
tool_choice: Tool selection preference.
system: System message content.
""" """
def __init__( def __init__(
@@ -441,6 +437,14 @@ class AnthropicLLMContext(OpenAILLMContext):
*, *,
system: Union[str, NotGiven] = NOT_GIVEN, system: Union[str, NotGiven] = NOT_GIVEN,
): ):
"""Initialize the Anthropic LLM context.
Args:
messages: Initial list of conversation messages.
tools: Available function calling tools.
tool_choice: Tool selection preference.
system: System message content.
"""
super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
# For beta prompt caching. This is a counter that tracks the number of turns # For beta prompt caching. This is a counter that tracks the number of turns

View File

@@ -1,10 +1,30 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""AssemblyAI WebSocket API message models and connection parameters.
This module defines Pydantic models for handling AssemblyAI's real-time
transcription WebSocket messages and connection configuration.
"""
from typing import List, Literal, Optional from typing import List, Literal, Optional
from pydantic import BaseModel, Field from pydantic import BaseModel, Field
class Word(BaseModel): class Word(BaseModel):
"""Represents a single word in a transcription with timing and confidence.""" """Represents a single word in a transcription with timing and confidence.
Parameters:
start: Start time of the word in milliseconds.
end: End time of the word in milliseconds.
text: The transcribed word text.
confidence: Confidence score for the word (0.0 to 1.0).
word_is_final: Whether this word is finalized and won't change.
"""
start: int start: int
end: int end: int
@@ -14,13 +34,23 @@ class Word(BaseModel):
class BaseMessage(BaseModel): class BaseMessage(BaseModel):
"""Base class for all AssemblyAI WebSocket messages.""" """Base class for all AssemblyAI WebSocket messages.
Parameters:
type: The message type identifier.
"""
type: str type: str
class BeginMessage(BaseMessage): class BeginMessage(BaseMessage):
"""Message sent when a new session begins.""" """Message sent when a new session begins.
Parameters:
type: Always "Begin" for this message type.
id: Unique session identifier.
expires_at: Unix timestamp when the session expires.
"""
type: Literal["Begin"] = "Begin" type: Literal["Begin"] = "Begin"
id: str id: str
@@ -28,7 +58,17 @@ class BeginMessage(BaseMessage):
class TurnMessage(BaseMessage): class TurnMessage(BaseMessage):
"""Message containing transcription data for a turn of speech.""" """Message containing transcription data for a turn of speech.
Parameters:
type: Always "Turn" for this message type.
turn_order: Sequential number of this turn in the session.
turn_is_formatted: Whether the transcript has been formatted.
end_of_turn: Whether this marks the end of a speaking turn.
transcript: The transcribed text for this turn.
end_of_turn_confidence: Confidence score for end-of-turn detection.
words: List of individual words with timing and confidence data.
"""
type: Literal["Turn"] = "Turn" type: Literal["Turn"] = "Turn"
turn_order: int turn_order: int
@@ -40,7 +80,13 @@ class TurnMessage(BaseMessage):
class TerminationMessage(BaseMessage): class TerminationMessage(BaseMessage):
"""Message sent when the session is terminated.""" """Message sent when the session is terminated.
Parameters:
type: Always "Termination" for this message type.
audio_duration_seconds: Total duration of audio processed.
session_duration_seconds: Total duration of the session.
"""
type: Literal["Termination"] = "Termination" type: Literal["Termination"] = "Termination"
audio_duration_seconds: float audio_duration_seconds: float
@@ -52,6 +98,18 @@ AnyMessage = BeginMessage | TurnMessage | TerminationMessage
class AssemblyAIConnectionParams(BaseModel): class AssemblyAIConnectionParams(BaseModel):
"""Configuration parameters for AssemblyAI WebSocket connection.
Parameters:
sample_rate: Audio sample rate in Hz. Defaults to 16000.
encoding: Audio encoding format. Defaults to "pcm_s16le".
formatted_finals: Whether to enable transcript formatting. Defaults to True.
word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds.
end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection.
min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn.
max_turn_silence: Maximum silence duration before forcing end-of-turn.
"""
sample_rate: int = 16000 sample_rate: int = 16000
encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le" encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le"
formatted_finals: bool = True formatted_finals: bool = True

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""AssemblyAI speech-to-text service implementation.
This module provides integration with AssemblyAI's real-time speech-to-text
WebSocket API for streaming audio transcription.
"""
import asyncio import asyncio
import json import json
from typing import Any, AsyncGenerator, Dict from typing import Any, AsyncGenerator, Dict
@@ -45,6 +51,13 @@ except ModuleNotFoundError as e:
class AssemblyAISTTService(STTService): class AssemblyAISTTService(STTService):
"""AssemblyAI real-time speech-to-text service.
Provides real-time speech transcription using AssemblyAI's WebSocket API.
Supports both interim and final transcriptions with configurable parameters
for audio processing and connection management.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -55,6 +68,16 @@ class AssemblyAISTTService(STTService):
vad_force_turn_endpoint: bool = True, vad_force_turn_endpoint: bool = True,
**kwargs, **kwargs,
): ):
"""Initialize the AssemblyAI STT service.
Args:
api_key: AssemblyAI API key for authentication.
language: Language code for transcription. Defaults to English (Language.EN).
api_endpoint_base_url: WebSocket endpoint URL. Defaults to AssemblyAI's streaming endpoint.
connection_params: Connection configuration parameters. Defaults to AssemblyAIConnectionParams().
vad_force_turn_endpoint: Whether to force turn endpoint on VAD stop. Defaults to True.
**kwargs: Additional arguments passed to parent STTService class.
"""
self._api_key = api_key self._api_key = api_key
self._language = language self._language = language
self._api_endpoint_base_url = api_endpoint_base_url self._api_endpoint_base_url = api_endpoint_base_url
@@ -75,22 +98,50 @@ class AssemblyAISTTService(STTService):
self._chunk_size_bytes = 0 self._chunk_size_bytes = 0
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
Returns:
True if metrics generation is supported.
"""
return True return True
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the speech-to-text service.
Args:
frame: Start frame to begin processing.
"""
await super().start(frame) await super().start(frame)
self._chunk_size_bytes = int(self._chunk_size_ms * self._sample_rate * 2 / 1000) self._chunk_size_bytes = int(self._chunk_size_ms * self._sample_rate * 2 / 1000)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the speech-to-text service.
Args:
frame: End frame to stop processing.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the speech-to-text service.
Args:
frame: Cancel frame to abort processing.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text conversion.
Args:
audio: Raw audio bytes to process.
Yields:
None (processing handled via WebSocket messages).
"""
self._audio_buffer.extend(audio) self._audio_buffer.extend(audio)
while len(self._audio_buffer) >= self._chunk_size_bytes: while len(self._audio_buffer) >= self._chunk_size_bytes:
@@ -101,6 +152,12 @@ class AssemblyAISTTService(STTService):
yield None yield None
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for VAD and metrics handling.
Args:
frame: Frame to process.
direction: Direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, UserStartedSpeakingFrame):
await self.start_ttfb_metrics() await self.start_ttfb_metrics()

View File

@@ -104,12 +104,6 @@ class AWSBedrockLLMContext(OpenAILLMContext):
Extends OpenAI LLM context to handle AWS Bedrock's specific message format Extends OpenAI LLM context to handle AWS Bedrock's specific message format
and system message handling. Manages conversion between OpenAI and Bedrock and system message handling. Manages conversion between OpenAI and Bedrock
message formats. message formats.
Args:
messages: List of conversation messages in OpenAI format.
tools: List of available function calling tools.
tool_choice: Tool selection strategy or specific tool choice.
system: System message content for AWS Bedrock.
""" """
def __init__( def __init__(
@@ -120,6 +114,14 @@ class AWSBedrockLLMContext(OpenAILLMContext):
*, *,
system: Optional[str] = None, system: Optional[str] = None,
): ):
"""Initialize AWS Bedrock LLM context.
Args:
messages: List of conversation messages in OpenAI format.
tools: List of available function calling tools.
tool_choice: Tool selection strategy or specific tool choice.
system: System message content for AWS Bedrock.
"""
super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
self.system = system self.system = system
@@ -656,16 +658,6 @@ class AWSBedrockLLMService(LLMService):
Provides inference capabilities for AWS Bedrock models including Amazon Nova Provides inference capabilities for AWS Bedrock models including Amazon Nova
and Anthropic Claude. Supports streaming responses, function calling, and and Anthropic Claude. Supports streaming responses, function calling, and
vision capabilities. vision capabilities.
Args:
model: The AWS Bedrock model identifier to use.
aws_access_key: AWS access key ID. If None, uses default credentials.
aws_secret_key: AWS secret access key. If None, uses default credentials.
aws_session_token: AWS session token for temporary credentials.
aws_region: AWS region for the Bedrock service.
params: Model parameters and configuration.
client_config: Custom boto3 client configuration.
**kwargs: Additional arguments passed to parent LLMService.
""" """
# Overriding the default adapter to use the Anthropic one. # Overriding the default adapter to use the Anthropic one.
@@ -702,6 +694,18 @@ class AWSBedrockLLMService(LLMService):
client_config: Optional[Config] = None, client_config: Optional[Config] = None,
**kwargs, **kwargs,
): ):
"""Initialize the AWS Bedrock LLM service.
Args:
model: The AWS Bedrock model identifier to use.
aws_access_key: AWS access key ID. If None, uses default credentials.
aws_secret_key: AWS secret access key. If None, uses default credentials.
aws_session_token: AWS session token for temporary credentials.
aws_region: AWS region for the Bedrock service.
params: Model parameters and configuration.
client_config: Custom boto3 client configuration.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
params = params or AWSBedrockLLMService.InputParams() params = params or AWSBedrockLLMService.InputParams()

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""AWS Transcribe Speech-to-Text service implementation.
This module provides a WebSocket-based connection to AWS Transcribe for real-time
speech-to-text transcription with support for multiple languages and audio formats.
"""
import asyncio import asyncio
import json import json
import os import os
@@ -37,6 +43,13 @@ except ModuleNotFoundError as e:
class AWSTranscribeSTTService(STTService): class AWSTranscribeSTTService(STTService):
"""AWS Transcribe Speech-to-Text service using WebSocket streaming.
Provides real-time speech transcription using AWS Transcribe's streaming API.
Supports multiple languages, configurable sample rates, and both interim and
final transcription results.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -48,6 +61,17 @@ class AWSTranscribeSTTService(STTService):
language: Language = Language.EN, language: Language = Language.EN,
**kwargs, **kwargs,
): ):
"""Initialize the AWS Transcribe STT service.
Args:
api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable.
aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable.
aws_session_token: AWS session token for temporary credentials. If None, uses AWS_SESSION_TOKEN environment variable.
region: AWS region for the service. Defaults to "us-east-1".
sample_rate: Audio sample rate in Hz. Must be 8000 or 16000. Defaults to 16000.
language: Language for transcription. Defaults to English.
**kwargs: Additional arguments passed to parent STTService class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._settings = { self._settings = {
@@ -79,14 +103,28 @@ class AWSTranscribeSTTService(STTService):
self._receive_task = None self._receive_task = None
def get_service_encoding(self, encoding: str) -> str: def get_service_encoding(self, encoding: str) -> str:
"""Convert internal encoding format to AWS Transcribe format.""" """Convert internal encoding format to AWS Transcribe format.
Args:
encoding: Internal encoding format string.
Returns:
AWS Transcribe compatible encoding format.
"""
encoding_map = { encoding_map = {
"linear16": "pcm", # AWS expects "pcm" for 16-bit linear PCM "linear16": "pcm", # AWS expects "pcm" for 16-bit linear PCM
} }
return encoding_map.get(encoding, encoding) return encoding_map.get(encoding, encoding)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Initialize the connection when the service starts.""" """Initialize the connection when the service starts.
Args:
frame: Start frame signaling service initialization.
Raises:
RuntimeError: If WebSocket connection cannot be established after retries.
"""
await super().start(frame) await super().start(frame)
logger.info("Starting AWS Transcribe service...") logger.info("Starting AWS Transcribe service...")
retry_count = 0 retry_count = 0
@@ -108,15 +146,32 @@ class AWSTranscribeSTTService(STTService):
raise RuntimeError("Failed to establish WebSocket connection after multiple attempts") raise RuntimeError("Failed to establish WebSocket connection after multiple attempts")
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the service and disconnect from AWS Transcribe.
Args:
frame: End frame signaling service shutdown.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the service and disconnect from AWS Transcribe.
Args:
frame: Cancel frame signaling service cancellation.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data and send to AWS Transcribe""" """Process audio data and send to AWS Transcribe.
Args:
audio: Raw audio bytes to transcribe.
Yields:
ErrorFrame: If processing fails or connection issues occur.
"""
try: try:
# Ensure WebSocket is connected # Ensure WebSocket is connected
if not self._ws_client or not self._ws_client.open: if not self._ws_client or not self._ws_client.open:
@@ -255,7 +310,14 @@ class AWSTranscribeSTTService(STTService):
self._ws_client = None self._ws_client = None
def language_to_service_language(self, language: Language) -> str | None: def language_to_service_language(self, language: Language) -> str | None:
"""Convert internal language enum to AWS Transcribe language code.""" """Convert internal language enum to AWS Transcribe language code.
Args:
language: Internal language enumeration value.
Returns:
AWS Transcribe compatible language code, or None if unsupported.
"""
language_map = { language_map = {
Language.EN: "en-US", Language.EN: "en-US",
Language.ES: "es-US", Language.ES: "es-US",

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""AWS Polly text-to-speech service implementation.
This module provides integration with Amazon Polly for text-to-speech synthesis,
supporting multiple languages, voices, and SSML features.
"""
import asyncio import asyncio
import os import os
from typing import AsyncGenerator, List, Optional from typing import AsyncGenerator, List, Optional
@@ -33,6 +39,14 @@ except ModuleNotFoundError as e:
def language_to_aws_language(language: Language) -> Optional[str]: def language_to_aws_language(language: Language) -> Optional[str]:
"""Convert a Language enum to AWS Polly language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding AWS Polly language code, or None if not supported.
"""
language_map = { language_map = {
# Arabic # Arabic
Language.AR: "arb", Language.AR: "arb",
@@ -109,7 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]:
class AWSPollyTTSService(TTSService): class AWSPollyTTSService(TTSService):
"""AWS Polly text-to-speech service.
Provides text-to-speech synthesis using Amazon Polly with support for
multiple languages, voices, SSML features, and voice customization
options including prosody controls.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for AWS Polly TTS configuration.
Parameters:
engine: TTS engine to use ('standard', 'neural', etc.).
language: Language for synthesis. Defaults to English.
pitch: Voice pitch adjustment (for standard engine only).
rate: Speech rate adjustment.
volume: Voice volume adjustment.
lexicon_names: List of pronunciation lexicons to apply.
"""
engine: Optional[str] = None engine: Optional[str] = None
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
pitch: Optional[str] = None pitch: Optional[str] = None
@@ -129,6 +161,18 @@ class AWSPollyTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initializes the AWS Polly TTS service.
Args:
api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable.
aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable.
aws_session_token: AWS session token for temporary credentials.
region: AWS region for Polly service. Defaults to 'us-east-1'.
voice_id: Voice ID to use for synthesis. Defaults to 'Joanna'.
sample_rate: Audio sample rate. If None, uses service default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AWSPollyTTSService.InputParams() params = params or AWSPollyTTSService.InputParams()
@@ -174,9 +218,22 @@ class AWSPollyTTSService(TTSService):
) )
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as AWS Polly service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to AWS Polly language format.
Args:
language: The language to convert.
Returns:
The AWS Polly-specific language code, or None if not supported.
"""
return language_to_aws_language(language) return language_to_aws_language(language)
def _construct_ssml(self, text: str) -> str: def _construct_ssml(self, text: str) -> str:
@@ -214,6 +271,15 @@ class AWSPollyTTSService(TTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using AWS Polly.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
def read_audio_data(**args): def read_audio_data(**args):
response = self._polly_client.synthesize_speech(**args) response = self._polly_client.synthesize_speech(**args)
if "AudioStream" in response: if "AudioStream" in response:
@@ -277,7 +343,14 @@ class AWSPollyTTSService(TTSService):
class PollyTTSService(AWSPollyTTSService): class PollyTTSService(AWSPollyTTSService):
"""Deprecated alias for AWSPollyTTSService."""
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the deprecated PollyTTSService.
Args:
**kwargs: All arguments passed to AWSPollyTTSService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
import warnings import warnings

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""AWS Transcribe utility functions and classes for WebSocket streaming.
This module provides utilities for creating presigned URLs, building event messages,
and handling AWS event stream protocol for real-time transcription services.
"""
import binascii import binascii
import datetime import datetime
import hashlib import hashlib
@@ -29,7 +35,31 @@ def get_presigned_url(
show_speaker_label: bool = False, show_speaker_label: bool = False,
enable_channel_identification: bool = False, enable_channel_identification: bool = False,
) -> str: ) -> str:
"""Create a presigned URL for AWS Transcribe streaming.""" """Create a presigned URL for AWS Transcribe streaming.
Args:
region: AWS region for the service.
credentials: Dictionary containing AWS credentials with keys:
- access_key: AWS access key ID
- secret_key: AWS secret access key
- session_token: AWS session token (optional)
language_code: Language code for transcription (e.g., "en-US").
media_encoding: Audio encoding format. Defaults to "pcm".
sample_rate: Audio sample rate in Hz. Defaults to 16000.
number_of_channels: Number of audio channels. Defaults to 1.
enable_partial_results_stabilization: Whether to enable partial result stabilization.
partial_results_stability: Stability level for partial results.
vocabulary_name: Custom vocabulary name to use.
vocabulary_filter_name: Vocabulary filter name to apply.
show_speaker_label: Whether to include speaker labels.
enable_channel_identification: Whether to enable channel identification.
Returns:
Presigned WebSocket URL for AWS Transcribe streaming.
Raises:
ValueError: If required AWS credentials are missing.
"""
access_key = credentials.get("access_key") access_key = credentials.get("access_key")
secret_key = credentials.get("secret_key") secret_key = credentials.get("secret_key")
session_token = credentials.get("session_token") session_token = credentials.get("session_token")
@@ -58,9 +88,23 @@ def get_presigned_url(
class AWSTranscribePresignedURL: class AWSTranscribePresignedURL:
"""Generator for AWS Transcribe presigned WebSocket URLs.
Handles AWS Signature Version 4 signing process to create authenticated
WebSocket URLs for streaming transcription requests.
"""
def __init__( def __init__(
self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1" self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1"
): ):
"""Initialize the presigned URL generator.
Args:
access_key: AWS access key ID.
secret_key: AWS secret access key.
session_token: AWS session token for temporary credentials.
region: AWS region for the service. Defaults to "us-east-1".
"""
self.access_key = access_key self.access_key = access_key
self.secret_key = secret_key self.secret_key = secret_key
self.session_token = session_token self.session_token = session_token
@@ -96,6 +140,23 @@ class AWSTranscribePresignedURL:
enable_partial_results_stabilization: bool = False, enable_partial_results_stabilization: bool = False,
partial_results_stability: str = "", partial_results_stability: str = "",
) -> str: ) -> str:
"""Generate a presigned WebSocket URL for AWS Transcribe.
Args:
sample_rate: Audio sample rate in Hz.
language_code: Language code for transcription.
media_encoding: Audio encoding format.
vocabulary_name: Custom vocabulary name.
vocabulary_filter_name: Vocabulary filter name.
show_speaker_label: Whether to include speaker labels.
enable_channel_identification: Whether to enable channel identification.
number_of_channels: Number of audio channels.
enable_partial_results_stabilization: Whether to enable partial result stabilization.
partial_results_stability: Stability level for partial results.
Returns:
Presigned WebSocket URL with authentication parameters.
"""
self.endpoint = f"wss://transcribestreaming.{self.region}.amazonaws.com:8443" self.endpoint = f"wss://transcribestreaming.{self.region}.amazonaws.com:8443"
self.host = f"transcribestreaming.{self.region}.amazonaws.com:8443" self.host = f"transcribestreaming.{self.region}.amazonaws.com:8443"
@@ -172,7 +233,15 @@ class AWSTranscribePresignedURL:
def get_headers(header_name: str, header_value: str) -> bytearray: def get_headers(header_name: str, header_value: str) -> bytearray:
"""Build a header following AWS event stream format.""" """Build a header following AWS event stream format.
Args:
header_name: Name of the header.
header_value: Value of the header.
Returns:
Encoded header as a bytearray following AWS event stream protocol.
"""
name = header_name.encode("utf-8") name = header_name.encode("utf-8")
name_byte_length = bytes([len(name)]) name_byte_length = bytes([len(name)])
value_type = bytes([7]) # 7 represents a string value_type = bytes([7]) # 7 represents a string
@@ -190,9 +259,21 @@ def get_headers(header_name: str, header_value: str) -> bytearray:
def build_event_message(payload: bytes) -> bytes: def build_event_message(payload: bytes) -> bytes:
""" """Build an event message for AWS Transcribe streaming.
Build an event message for AWS Transcribe streaming.
Matches AWS sample: https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py Creates a properly formatted AWS event stream message containing audio data
for real-time transcription. Follows the AWS event stream protocol with
prelude, headers, payload, and CRC checksums.
Args:
payload: Raw audio bytes to include in the event message.
Returns:
Complete event message as bytes, ready to send via WebSocket.
Note:
Implementation matches AWS sample:
https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py
""" """
# Build headers # Build headers
content_type_header = get_headers(":content-type", "application/octet-stream") content_type_header = get_headers(":content-type", "application/octet-stream")
@@ -235,6 +316,22 @@ def build_event_message(payload: bytes) -> bytes:
def decode_event(message): def decode_event(message):
"""Decode an AWS event stream message.
Parses an AWS event stream message to extract headers and payload,
verifying CRC checksums for data integrity.
Args:
message: Raw event stream message bytes received from AWS.
Returns:
Tuple containing:
- Dictionary of parsed headers
- Dictionary of parsed JSON payload
Raises:
AssertionError: If CRC checksum verification fails.
"""
# Extract the prelude, headers, payload and CRC # Extract the prelude, headers, payload and CRC
prelude = message[:8] prelude = message[:8]
total_length, headers_length = struct.unpack(">II", prelude) total_length, headers_length = struct.unpack(">II", prelude)

View File

@@ -96,7 +96,13 @@ class AWSNovaSonicUnhandledFunctionException(Exception):
class ContentType(Enum): class ContentType(Enum):
"""Content types supported by AWS Nova Sonic.""" """Content types supported by AWS Nova Sonic.
Parameters:
AUDIO: Audio content type.
TEXT: Text content type.
TOOL: Tool content type.
"""
AUDIO = "AUDIO" AUDIO = "AUDIO"
TEXT = "TEXT" TEXT = "TEXT"
@@ -104,7 +110,12 @@ class ContentType(Enum):
class TextStage(Enum): class TextStage(Enum):
"""Text generation stages in AWS Nova Sonic responses.""" """Text generation stages in AWS Nova Sonic responses.
Parameters:
FINAL: Final text that has been fully generated.
SPECULATIVE: Speculative text that is still being generated.
"""
FINAL = "FINAL" # what has been said FINAL = "FINAL" # what has been said
SPECULATIVE = "SPECULATIVE" # what's planned to be said SPECULATIVE = "SPECULATIVE" # what's planned to be said
@@ -127,6 +138,7 @@ class CurrentContent:
text_content: str # starts as None, then fills in if text text_content: str # starts as None, then fills in if text
def __str__(self): def __str__(self):
"""String representation of the current content."""
return ( return (
f"CurrentContent(\n" f"CurrentContent(\n"
f" type={self.type.name},\n" f" type={self.type.name},\n"
@@ -172,18 +184,6 @@ class AWSNovaSonicLLMService(LLMService):
Provides bidirectional audio streaming, real-time transcription, text generation, Provides bidirectional audio streaming, real-time transcription, text generation,
and function calling capabilities using AWS Nova Sonic model. and function calling capabilities using AWS Nova Sonic model.
Args:
secret_access_key: AWS secret access key for authentication.
access_key_id: AWS access key ID for authentication.
region: AWS region where the service is hosted.
model: Model identifier. Defaults to "amazon.nova-sonic-v1:0".
voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy.
params: Model parameters for audio configuration and inference.
system_instruction: System-level instruction for the model.
tools: Available tools/functions for the model to use.
send_transcription_frames: Whether to emit transcription frames.
**kwargs: Additional arguments passed to the parent LLMService.
""" """
# Override the default adapter to use the AWSNovaSonicLLMAdapter one # Override the default adapter to use the AWSNovaSonicLLMAdapter one
@@ -203,6 +203,20 @@ class AWSNovaSonicLLMService(LLMService):
send_transcription_frames: bool = True, send_transcription_frames: bool = True,
**kwargs, **kwargs,
): ):
"""Initializes the AWS Nova Sonic LLM service.
Args:
secret_access_key: AWS secret access key for authentication.
access_key_id: AWS access key ID for authentication.
region: AWS region where the service is hosted.
model: Model identifier. Defaults to "amazon.nova-sonic-v1:0".
voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy.
params: Model parameters for audio configuration and inference.
system_instruction: System-level instruction for the model.
tools: Available tools/functions for the model to use.
send_transcription_frames: Whether to emit transcription frames.
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._secret_access_key = secret_access_key self._secret_access_key = secret_access_key
self._access_key_id = access_key_id self._access_key_id = access_key_id

View File

@@ -41,7 +41,14 @@ from pipecat.services.openai.llm import (
class Role(Enum): class Role(Enum):
"""Roles supported in AWS Nova Sonic conversations.""" """Roles supported in AWS Nova Sonic conversations.
Parameters:
SYSTEM: System-level messages (not used in conversation history).
USER: Messages sent by the user.
ASSISTANT: Messages sent by the assistant.
TOOL: Messages sent by tools (not used in conversation history).
"""
SYSTEM = "SYSTEM" SYSTEM = "SYSTEM"
USER = "USER" USER = "USER"
@@ -80,14 +87,16 @@ class AWSNovaSonicLLMContext(OpenAILLMContext):
Extends OpenAI context with Nova Sonic-specific message handling, Extends OpenAI context with Nova Sonic-specific message handling,
conversation history management, and text buffering capabilities. conversation history management, and text buffering capabilities.
Args:
messages: Initial messages for the context.
tools: Available tools for the context.
**kwargs: Additional arguments passed to parent class.
""" """
def __init__(self, messages=None, tools=None, **kwargs): def __init__(self, messages=None, tools=None, **kwargs):
"""Initialize AWS Nova Sonic LLM context.
Args:
messages: Initial messages for the context.
tools: Available tools for the context.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(messages=messages, tools=tools, **kwargs) super().__init__(messages=messages, tools=tools, **kwargs)
self.__setup_local() self.__setup_local()

View File

@@ -4,14 +4,22 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
from typing import Optional """Language conversion utilities for Azure services."""
from loguru import logger from typing import Optional
from pipecat.transcriptions.language import Language from pipecat.transcriptions.language import Language
def language_to_azure_language(language: Language) -> Optional[str]: def language_to_azure_language(language: Language) -> Optional[str]:
"""Convert a Language enum to Azure language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding Azure language code, or None if not supported.
"""
language_map = { language_map = {
# Afrikaans # Afrikaans
Language.AF: "af-ZA", Language.AF: "af-ZA",

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Azure OpenAI image generation service implementation.
This module provides integration with Azure's OpenAI image generation API
using REST endpoints for creating images from text prompts.
"""
import asyncio import asyncio
import io import io
from typing import AsyncGenerator from typing import AsyncGenerator
@@ -17,6 +23,13 @@ from pipecat.services.image_service import ImageGenService
class AzureImageGenServiceREST(ImageGenService): class AzureImageGenServiceREST(ImageGenService):
"""Azure OpenAI REST-based image generation service.
Provides image generation using Azure's OpenAI service via REST API.
Supports asynchronous image generation with polling for completion
and automatic image download and processing.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -27,6 +40,16 @@ class AzureImageGenServiceREST(ImageGenService):
aiohttp_session: aiohttp.ClientSession, aiohttp_session: aiohttp.ClientSession,
api_version="2023-06-01-preview", api_version="2023-06-01-preview",
): ):
"""Initialize the AzureImageGenServiceREST.
Args:
image_size: Size specification for generated images (e.g., "1024x1024").
api_key: Azure OpenAI API key for authentication.
endpoint: Azure OpenAI endpoint URL.
model: The image generation model to use.
aiohttp_session: Shared aiohttp session for HTTP requests.
api_version: Azure API version string. Defaults to "2023-06-01-preview".
"""
super().__init__() super().__init__()
self._api_key = api_key self._api_key = api_key
@@ -37,6 +60,15 @@ class AzureImageGenServiceREST(ImageGenService):
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
"""Generate an image from a text prompt using Azure OpenAI.
Args:
prompt: The text prompt describing the desired image.
Yields:
URLImageRawFrame containing the generated image data, or
ErrorFrame if generation fails.
"""
url = f"{self._azure_endpoint}openai/images/generations:submit?api-version={self._api_version}" url = f"{self._azure_endpoint}openai/images/generations:submit?api-version={self._api_version}"
headers = {"api-key": self._api_key, "Content-Type": "application/json"} headers = {"api-key": self._api_key, "Content-Type": "application/json"}

View File

@@ -17,13 +17,6 @@ class AzureLLMService(OpenAILLMService):
This service extends OpenAILLMService to connect to Azure's OpenAI endpoint while This service extends OpenAILLMService to connect to Azure's OpenAI endpoint while
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Args:
api_key: The API key for accessing Azure OpenAI.
endpoint: The Azure endpoint URL.
model: The model identifier to use.
api_version: Azure API version. Defaults to "2024-09-01-preview".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -35,6 +28,15 @@ class AzureLLMService(OpenAILLMService):
api_version: str = "2024-09-01-preview", api_version: str = "2024-09-01-preview",
**kwargs, **kwargs,
): ):
"""Initialize the Azure LLM service.
Args:
api_key: The API key for accessing Azure OpenAI.
endpoint: The Azure endpoint URL.
model: The model identifier to use.
api_version: Azure API version. Defaults to "2024-09-01-preview".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
# Initialize variables before calling parent __init__() because that # Initialize variables before calling parent __init__() because that
# will call create_client() and we need those values there. # will call create_client() and we need those values there.
self._endpoint = endpoint self._endpoint = endpoint

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Azure Speech-to-Text service implementation for Pipecat.
This module provides speech-to-text functionality using Azure Cognitive Services
Speech SDK for real-time audio transcription.
"""
import asyncio import asyncio
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
@@ -40,6 +46,13 @@ except ModuleNotFoundError as e:
class AzureSTTService(STTService): class AzureSTTService(STTService):
"""Azure Speech-to-Text service for real-time audio transcription.
This service uses Azure Cognitive Services Speech SDK to convert speech
audio into text transcriptions. It supports continuous recognition and
provides real-time transcription results with timing information.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -49,6 +62,15 @@ class AzureSTTService(STTService):
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Azure STT service.
Args:
api_key: Azure Cognitive Services subscription key.
region: Azure region for the Speech service (e.g., 'eastus').
language: Language for speech recognition. Defaults to English (US).
sample_rate: Audio sample rate in Hz. If None, uses service default.
**kwargs: Additional arguments passed to parent STTService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
self._speech_config = SpeechConfig( self._speech_config = SpeechConfig(
@@ -66,9 +88,25 @@ class AzureSTTService(STTService):
} }
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate performance metrics.
Returns:
True as this service supports metrics generation.
"""
return True return True
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text conversion.
Feeds audio data to the Azure speech recognizer for processing.
Recognition results are handled asynchronously through callbacks.
Args:
audio: Raw audio bytes to process.
Yields:
None - actual transcription frames are pushed via callbacks.
"""
await self.start_processing_metrics() await self.start_processing_metrics()
await self.start_ttfb_metrics() await self.start_ttfb_metrics()
if self._audio_stream: if self._audio_stream:
@@ -76,6 +114,14 @@ class AzureSTTService(STTService):
yield None yield None
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the speech recognition service.
Initializes the Azure speech recognizer with audio stream configuration
and begins continuous speech recognition.
Args:
frame: Frame indicating the start of processing.
"""
await super().start(frame) await super().start(frame)
if self._audio_stream: if self._audio_stream:
@@ -93,6 +139,13 @@ class AzureSTTService(STTService):
self._speech_recognizer.start_continuous_recognition_async() self._speech_recognizer.start_continuous_recognition_async()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the speech recognition service.
Cleanly shuts down the Azure speech recognizer and closes audio streams.
Args:
frame: Frame indicating the end of processing.
"""
await super().stop(frame) await super().stop(frame)
if self._speech_recognizer: if self._speech_recognizer:
@@ -102,6 +155,13 @@ class AzureSTTService(STTService):
self._audio_stream.close() self._audio_stream.close()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the speech recognition service.
Immediately stops recognition and closes resources.
Args:
frame: Frame indicating cancellation.
"""
await super().cancel(frame) await super().cancel(frame)
if self._speech_recognizer: if self._speech_recognizer:

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Azure Cognitive Services Text-to-Speech service implementations."""
import asyncio import asyncio
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
@@ -39,6 +41,15 @@ except ModuleNotFoundError as e:
def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputFormat: def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputFormat:
"""Convert sample rate to Azure speech synthesis output format.
Args:
sample_rate: Sample rate in Hz.
Returns:
Corresponding Azure SpeechSynthesisOutputFormat enum value.
Defaults to Raw24Khz16BitMonoPcm if sample rate not found.
"""
sample_rate_map = { sample_rate_map = {
8000: SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm, 8000: SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm,
16000: SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm, 16000: SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm,
@@ -51,7 +62,26 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma
class AzureBaseTTSService(TTSService): class AzureBaseTTSService(TTSService):
"""Base class for Azure Cognitive Services text-to-speech implementations.
Provides common functionality for Azure TTS services including SSML
construction, voice configuration, and parameter management.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Azure TTS voice configuration.
Parameters:
emphasis: Emphasis level for speech ("strong", "moderate", "reduced").
language: Language for synthesis. Defaults to English (US).
pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high").
rate: Speech rate multiplier. Defaults to "1.05".
role: Voice role for expression (e.g., "YoungAdultFemale").
style: Speaking style (e.g., "cheerful", "sad", "excited").
style_degree: Intensity of the speaking style (0.01 to 2.0).
volume: Volume level (e.g., "+20%", "loud", "x-soft").
"""
emphasis: Optional[str] = None emphasis: Optional[str] = None
language: Optional[Language] = Language.EN_US language: Optional[Language] = Language.EN_US
pitch: Optional[str] = None pitch: Optional[str] = None
@@ -71,6 +101,16 @@ class AzureBaseTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Azure TTS service with configuration parameters.
Args:
api_key: Azure Cognitive Services subscription key.
region: Azure region identifier (e.g., "eastus", "westus2").
voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural".
sample_rate: Audio sample rate in Hz. If None, uses service default.
params: Voice and synthesis parameters configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or AzureBaseTTSService.InputParams() params = params or AzureBaseTTSService.InputParams()
@@ -94,9 +134,22 @@ class AzureBaseTTSService(TTSService):
self._speech_synthesizer = None self._speech_synthesizer = None
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Azure TTS service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Azure language format.
Args:
language: The language to convert.
Returns:
The Azure-specific language code, or None if not supported.
"""
return language_to_azure_language(language) return language_to_azure_language(language)
def _construct_ssml(self, text: str) -> str: def _construct_ssml(self, text: str) -> str:
@@ -146,13 +199,30 @@ class AzureBaseTTSService(TTSService):
class AzureTTSService(AzureBaseTTSService): class AzureTTSService(AzureBaseTTSService):
"""Azure Cognitive Services streaming TTS service.
Provides real-time text-to-speech synthesis using Azure's WebSocket-based
streaming API. Audio chunks are streamed as they become available for
lower latency playback.
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the Azure streaming TTS service.
Args:
**kwargs: All arguments passed to AzureBaseTTSService parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._speech_config = None self._speech_config = None
self._speech_synthesizer = None self._speech_synthesizer = None
self._audio_queue = asyncio.Queue() self._audio_queue = asyncio.Queue()
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Azure TTS service and initialize speech synthesizer.
Args:
frame: Start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
if self._speech_config: if self._speech_config:
@@ -183,24 +253,33 @@ class AzureTTSService(AzureBaseTTSService):
self._speech_synthesizer.synthesis_canceled.connect(self._handle_canceled) self._speech_synthesizer.synthesis_canceled.connect(self._handle_canceled)
def _handle_synthesizing(self, evt): def _handle_synthesizing(self, evt):
"""Handle audio chunks as they arrive""" """Handle audio chunks as they arriv."""
if evt.result and evt.result.audio_data: if evt.result and evt.result.audio_data:
self._audio_queue.put_nowait(evt.result.audio_data) self._audio_queue.put_nowait(evt.result.audio_data)
def _handle_completed(self, evt): def _handle_completed(self, evt):
"""Handle synthesis completion""" """Handle synthesis completion."""
self._audio_queue.put_nowait(None) # Signal completion self._audio_queue.put_nowait(None) # Signal completion
def _handle_canceled(self, evt): def _handle_canceled(self, evt):
"""Handle synthesis cancellation""" """Handle synthesis cancellation."""
logger.error(f"Speech synthesis canceled: {evt.result.cancellation_details.reason}") logger.error(f"Speech synthesis canceled: {evt.result.cancellation_details.reason}")
self._audio_queue.put_nowait(None) self._audio_queue.put_nowait(None)
async def flush_audio(self): async def flush_audio(self):
"""Flush any pending audio data."""
logger.trace(f"{self}: flushing audio") logger.trace(f"{self}: flushing audio")
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Azure's streaming synthesis.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing synthesized speech data.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:
@@ -244,12 +323,29 @@ class AzureTTSService(AzureBaseTTSService):
class AzureHttpTTSService(AzureBaseTTSService): class AzureHttpTTSService(AzureBaseTTSService):
"""Azure Cognitive Services HTTP-based TTS service.
Provides text-to-speech synthesis using Azure's HTTP API for simpler,
non-streaming synthesis. Suitable for use cases where streaming is not
required and simpler integration is preferred.
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the Azure HTTP TTS service.
Args:
**kwargs: All arguments passed to AzureBaseTTSService parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._speech_config = None self._speech_config = None
self._speech_synthesizer = None self._speech_synthesizer = None
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Azure HTTP TTS service and initialize speech synthesizer.
Args:
frame: Start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
if self._speech_config: if self._speech_config:
@@ -269,6 +365,14 @@ class AzureHttpTTSService(AzureBaseTTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Azure's HTTP synthesis API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the complete synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
await self.start_ttfb_metrics() await self.start_ttfb_metrics()

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Cartesia Speech-to-Text service implementation.
This module provides a WebSocket-based STT service that integrates with
the Cartesia Live transcription API for real-time speech recognition.
"""
import asyncio import asyncio
import json import json
import urllib.parse import urllib.parse
@@ -30,6 +36,12 @@ from pipecat.utils.tracing.service_decorators import traced_stt
class CartesiaLiveOptions: class CartesiaLiveOptions:
"""Configuration options for Cartesia Live STT service.
Manages transcription parameters including model selection, language,
audio encoding format, and sample rate settings.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -39,6 +51,15 @@ class CartesiaLiveOptions:
sample_rate: int = 16000, sample_rate: int = 16000,
**kwargs, **kwargs,
): ):
"""Initialize CartesiaLiveOptions with default or provided parameters.
Args:
model: The transcription model to use. Defaults to "ink-whisper".
language: Target language for transcription. Defaults to English.
encoding: Audio encoding format. Defaults to "pcm_s16le".
sample_rate: Audio sample rate in Hz. Defaults to 16000.
**kwargs: Additional parameters for the transcription service.
"""
self.model = model self.model = model
self.language = language self.language = language
self.encoding = encoding self.encoding = encoding
@@ -46,6 +67,11 @@ class CartesiaLiveOptions:
self.additional_params = kwargs self.additional_params = kwargs
def to_dict(self): def to_dict(self):
"""Convert options to dictionary format.
Returns:
Dictionary containing all configuration parameters.
"""
params = { params = {
"model": self.model, "model": self.model,
"language": self.language if isinstance(self.language, str) else self.language.value, "language": self.language if isinstance(self.language, str) else self.language.value,
@@ -56,19 +82,48 @@ class CartesiaLiveOptions:
return params return params
def items(self): def items(self):
"""Get configuration items as key-value pairs.
Returns:
Iterator of (key, value) tuples for all configuration parameters.
"""
return self.to_dict().items() return self.to_dict().items()
def get(self, key, default=None): def get(self, key, default=None):
"""Get a configuration value by key.
Args:
key: The configuration parameter name to retrieve.
default: Default value if key is not found.
Returns:
The configuration value or default if not found.
"""
if hasattr(self, key): if hasattr(self, key):
return getattr(self, key) return getattr(self, key)
return self.additional_params.get(key, default) return self.additional_params.get(key, default)
@classmethod @classmethod
def from_json(cls, json_str: str) -> "CartesiaLiveOptions": def from_json(cls, json_str: str) -> "CartesiaLiveOptions":
"""Create options from JSON string.
Args:
json_str: JSON string containing configuration parameters.
Returns:
New CartesiaLiveOptions instance with parsed parameters.
"""
return cls(**json.loads(json_str)) return cls(**json.loads(json_str))
class CartesiaSTTService(STTService): class CartesiaSTTService(STTService):
"""Speech-to-text service using Cartesia Live API.
Provides real-time speech transcription through WebSocket connection
to Cartesia's Live transcription service. Supports both interim and
final transcriptions with configurable models and languages.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -78,6 +133,15 @@ class CartesiaSTTService(STTService):
live_options: Optional[CartesiaLiveOptions] = None, live_options: Optional[CartesiaLiveOptions] = None,
**kwargs, **kwargs,
): ):
"""Initialize CartesiaSTTService with API key and options.
Args:
api_key: Authentication key for Cartesia API.
base_url: Custom API endpoint URL. If empty, uses default.
sample_rate: Audio sample rate in Hz. Defaults to 16000.
live_options: Configuration options for transcription service.
**kwargs: Additional arguments passed to parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None) sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
@@ -108,21 +172,49 @@ class CartesiaSTTService(STTService):
self._receiver_task = None self._receiver_task = None
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
Returns:
True, indicating metrics are supported.
"""
return True return True
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the STT service and establish connection.
Args:
frame: Frame indicating service should start.
"""
await super().start(frame) await super().start(frame)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the STT service and close connection.
Args:
frame: Frame indicating service should stop.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the STT service and close connection.
Args:
frame: Frame indicating service should be cancelled.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process audio data for speech-to-text transcription.
Args:
audio: Raw audio bytes to transcribe.
Yields:
None - transcription results are handled via WebSocket responses.
"""
# If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again # If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again
if not self._connection or self._connection.closed: if not self._connection or self._connection.closed:
await self._connect() await self._connect()
@@ -225,10 +317,17 @@ class CartesiaSTTService(STTService):
self._connection = None self._connection = None
async def start_metrics(self): async def start_metrics(self):
"""Start performance metrics collection for transcription processing."""
await self.start_ttfb_metrics() await self.start_ttfb_metrics()
await self.start_processing_metrics() await self.start_processing_metrics()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process incoming frames and handle speech events.
Args:
frame: The frame to process.
direction: Direction of frame flow in the pipeline.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, UserStartedSpeakingFrame):

View File

@@ -90,19 +90,6 @@ class CartesiaTTSService(AudioContextWordTTSService):
Provides text-to-speech using Cartesia's streaming WebSocket API. Provides text-to-speech using Cartesia's streaming WebSocket API.
Supports word-level timestamps, audio context management, and various voice Supports word-level timestamps, audio context management, and various voice
customization options including speed and emotion controls. customization options including speed and emotion controls.
Args:
api_key: Cartesia API key for authentication.
voice_id: ID of the voice to use for synthesis.
cartesia_version: API version string for Cartesia service.
url: WebSocket URL for Cartesia TTS API.
model: TTS model to use (e.g., "sonic-2").
sample_rate: Audio sample rate. If None, uses default.
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
text_aggregator: Custom text aggregator for processing input text.
**kwargs: Additional arguments passed to the parent service.
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
@@ -133,6 +120,21 @@ class CartesiaTTSService(AudioContextWordTTSService):
text_aggregator: Optional[BaseTextAggregator] = None, text_aggregator: Optional[BaseTextAggregator] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Cartesia TTS service.
Args:
api_key: Cartesia API key for authentication.
voice_id: ID of the voice to use for synthesis.
cartesia_version: API version string for Cartesia service.
url: WebSocket URL for Cartesia TTS API.
model: TTS model to use (e.g., "sonic-2").
sample_rate: Audio sample rate. If None, uses default.
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
text_aggregator: Custom text aggregator for processing input text.
**kwargs: Additional arguments passed to the parent service.
"""
# Aggregating sentences still gives cleaner-sounding results and fewer # Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for a # artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama # full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
@@ -404,18 +406,6 @@ class CartesiaHttpTTSService(TTSService):
Provides text-to-speech using Cartesia's HTTP API for simpler, non-streaming Provides text-to-speech using Cartesia's HTTP API for simpler, non-streaming
synthesis. Suitable for use cases where streaming is not required and simpler synthesis. Suitable for use cases where streaming is not required and simpler
integration is preferred. integration is preferred.
Args:
api_key: Cartesia API key for authentication.
voice_id: ID of the voice to use for synthesis.
model: TTS model to use (e.g., "sonic-2").
base_url: Base URL for Cartesia HTTP API.
cartesia_version: API version string for Cartesia service.
sample_rate: Audio sample rate. If None, uses default.
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
@@ -445,6 +435,20 @@ class CartesiaHttpTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Cartesia HTTP TTS service.
Args:
api_key: Cartesia API key for authentication.
voice_id: ID of the voice to use for synthesis.
model: TTS model to use (e.g., "sonic-2").
base_url: Base URL for Cartesia HTTP API.
cartesia_version: API version string for Cartesia service.
sample_rate: Audio sample rate. If None, uses default.
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or CartesiaHttpTTSService.InputParams() params = params or CartesiaHttpTTSService.InputParams()

View File

@@ -21,12 +21,6 @@ class CerebrasLLMService(OpenAILLMService):
This service extends OpenAILLMService to connect to Cerebras's API endpoint while This service extends OpenAILLMService to connect to Cerebras's API endpoint while
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Args:
api_key: The API key for accessing Cerebras's API.
base_url: The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1".
model: The model identifier to use. Defaults to "llama-3.3-70b".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -37,6 +31,14 @@ class CerebrasLLMService(OpenAILLMService):
model: str = "llama-3.3-70b", model: str = "llama-3.3-70b",
**kwargs, **kwargs,
): ):
"""Initialize the Cerebras LLM service.
Args:
api_key: The API key for accessing Cerebras's API.
base_url: The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1".
model: The model identifier to use. Defaults to "llama-3.3-70b".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
def create_client(self, api_key=None, base_url=None, **kwargs): def create_client(self, api_key=None, base_url=None, **kwargs):

View File

@@ -48,15 +48,6 @@ class DeepgramSTTService(STTService):
Provides real-time speech recognition using Deepgram's WebSocket API. Provides real-time speech recognition using Deepgram's WebSocket API.
Supports configurable models, languages, VAD events, and various audio Supports configurable models, languages, VAD events, and various audio
processing options. processing options.
Args:
api_key: Deepgram API key for authentication.
url: Deprecated. Use base_url instead.
base_url: Custom Deepgram API base URL.
sample_rate: Audio sample rate. If None, uses default or live_options value.
live_options: Deepgram LiveOptions for detailed configuration.
addons: Additional Deepgram features to enable.
**kwargs: Additional arguments passed to the parent STTService.
""" """
def __init__( def __init__(
@@ -70,6 +61,17 @@ class DeepgramSTTService(STTService):
addons: Optional[Dict] = None, addons: Optional[Dict] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Deepgram STT service.
Args:
api_key: Deepgram API key for authentication.
url: Deprecated. Use base_url instead.
base_url: Custom Deepgram API base URL.
sample_rate: Audio sample rate. If None, uses default or live_options value.
live_options: Deepgram LiveOptions for detailed configuration.
addons: Additional Deepgram features to enable.
**kwargs: Additional arguments passed to the parent STTService.
"""
sample_rate = sample_rate or (live_options.sample_rate if live_options else None) sample_rate = sample_rate or (live_options.sample_rate if live_options else None)
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Deepgram text-to-speech service implementation.
This module provides integration with Deepgram's text-to-speech API
for generating speech from text using various voice models.
"""
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
from loguru import logger from loguru import logger
@@ -27,6 +33,13 @@ except ModuleNotFoundError as e:
class DeepgramTTSService(TTSService): class DeepgramTTSService(TTSService):
"""Deepgram text-to-speech service.
Provides text-to-speech synthesis using Deepgram's streaming API.
Supports various voice models and audio encoding formats with
configurable sample rates and quality settings.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -37,6 +50,16 @@ class DeepgramTTSService(TTSService):
encoding: str = "linear16", encoding: str = "linear16",
**kwargs, **kwargs,
): ):
"""Initialize the Deepgram TTS service.
Args:
api_key: Deepgram API key for authentication.
voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en".
base_url: Custom base URL for Deepgram API. Uses default if empty.
sample_rate: Audio sample rate in Hz. If None, uses service default.
encoding: Audio encoding format. Defaults to "linear16".
**kwargs: Additional arguments passed to parent TTSService class.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
self._settings = { self._settings = {
@@ -48,10 +71,23 @@ class DeepgramTTSService(TTSService):
self._deepgram_client = DeepgramClient(api_key, config=client_options) self._deepgram_client = DeepgramClient(api_key, config=client_options)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
Returns:
True, as Deepgram TTS service supports metrics generation.
"""
return True return True
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Deepgram's TTS API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech, plus start/stop frames.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
options = SpeakOptions( options = SpeakOptions(

View File

@@ -21,12 +21,6 @@ class DeepSeekLLMService(OpenAILLMService):
This service extends OpenAILLMService to connect to DeepSeek's API endpoint while This service extends OpenAILLMService to connect to DeepSeek's API endpoint while
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Args:
api_key: The API key for accessing DeepSeek's API.
base_url: The base URL for DeepSeek API. Defaults to "https://api.deepseek.com/v1".
model: The model identifier to use. Defaults to "deepseek-chat".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -37,6 +31,14 @@ class DeepSeekLLMService(OpenAILLMService):
model: str = "deepseek-chat", model: str = "deepseek-chat",
**kwargs, **kwargs,
): ):
"""Initialize the DeepSeek LLM service.
Args:
api_key: The API key for accessing DeepSeek's API.
base_url: The base URL for DeepSeek API. Defaults to "https://api.deepseek.com/v1".
model: The model identifier to use. Defaults to "deepseek-chat".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
def create_client(self, api_key=None, base_url=None, **kwargs): def create_client(self, api_key=None, base_url=None, **kwargs):

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""ElevenLabs text-to-speech service implementations.
This module provides WebSocket and HTTP-based TTS services using ElevenLabs API
with support for streaming audio, word timestamps, and voice customization.
"""
import asyncio import asyncio
import base64 import base64
import json import json
@@ -57,6 +63,14 @@ ELEVENLABS_MULTILINGUAL_MODELS = {
def language_to_elevenlabs_language(language: Language) -> Optional[str]: def language_to_elevenlabs_language(language: Language) -> Optional[str]:
"""Convert a Language enum to ElevenLabs language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding ElevenLabs language code, or None if not supported.
"""
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.AR: "ar", Language.AR: "ar",
Language.BG: "bg", Language.BG: "bg",
@@ -106,6 +120,14 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]:
def output_format_from_sample_rate(sample_rate: int) -> str: def output_format_from_sample_rate(sample_rate: int) -> str:
"""Get the appropriate output format string for a given sample rate.
Args:
sample_rate: The audio sample rate in Hz.
Returns:
The ElevenLabs output format string.
"""
match sample_rate: match sample_rate:
case 8000: case 8000:
return "pcm_8000" return "pcm_8000"
@@ -129,10 +151,10 @@ def build_elevenlabs_voice_settings(
"""Build voice settings dictionary for ElevenLabs based on provided settings. """Build voice settings dictionary for ElevenLabs based on provided settings.
Args: Args:
settings: Dictionary containing voice settings parameters settings: Dictionary containing voice settings parameters.
Returns: Returns:
Dictionary of voice settings or None if no valid settings are provided Dictionary of voice settings or None if no valid settings are provided.
""" """
voice_setting_keys = ["stability", "similarity_boost", "style", "use_speaker_boost", "speed"] voice_setting_keys = ["stability", "similarity_boost", "style", "use_speaker_boost", "speed"]
@@ -147,6 +169,15 @@ def build_elevenlabs_voice_settings(
def calculate_word_times( def calculate_word_times(
alignment_info: Mapping[str, Any], cumulative_time: float alignment_info: Mapping[str, Any], cumulative_time: float
) -> List[Tuple[str, float]]: ) -> List[Tuple[str, float]]:
"""Calculate word timestamps from character alignment information.
Args:
alignment_info: Character alignment data from ElevenLabs API.
cumulative_time: Base time offset for this chunk.
Returns:
List of (word, timestamp) tuples.
"""
zipped_times = list(zip(alignment_info["chars"], alignment_info["charStartTimesMs"])) zipped_times = list(zip(alignment_info["chars"], alignment_info["charStartTimesMs"]))
words = "".join(alignment_info["chars"]).split(" ") words = "".join(alignment_info["chars"]).split(" ")
@@ -166,7 +197,28 @@ def calculate_word_times(
class ElevenLabsTTSService(AudioContextWordTTSService): class ElevenLabsTTSService(AudioContextWordTTSService):
"""ElevenLabs WebSocket-based TTS service with word timestamps.
Provides real-time text-to-speech using ElevenLabs' WebSocket streaming API.
Supports word-level timestamps, audio context management, and various voice
customization options including stability, similarity boost, and speed controls.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for ElevenLabs TTS configuration.
Parameters:
language: Language to use for synthesis.
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.25 to 4.0).
auto_mode: Whether to enable automatic mode optimization.
enable_ssml_parsing: Whether to parse SSML tags in text.
enable_logging: Whether to enable ElevenLabs logging.
"""
language: Optional[Language] = None language: Optional[Language] = None
stability: Optional[float] = None stability: Optional[float] = None
similarity_boost: Optional[float] = None similarity_boost: Optional[float] = None
@@ -188,6 +240,17 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the ElevenLabs TTS service.
Args:
api_key: ElevenLabs API key for authentication.
voice_id: ID of the voice to use for synthesis.
model: TTS model to use (e.g., "eleven_flash_v2_5").
url: WebSocket URL for ElevenLabs TTS API.
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent service.
"""
# Aggregating sentences still gives cleaner-sounding results and fewer # Aggregating sentences still gives cleaner-sounding results and fewer
# artifacts than streaming one word at a time. On average, waiting for a # artifacts than streaming one word at a time. On average, waiting for a
# full sentence should only "cost" us 15ms or so with GPT-4o or a Llama # full sentence should only "cost" us 15ms or so with GPT-4o or a Llama
@@ -244,21 +307,40 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
self._keepalive_task = None self._keepalive_task = None
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as ElevenLabs service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to ElevenLabs language format.
Args:
language: The language to convert.
Returns:
The ElevenLabs-specific language code, or None if not supported.
"""
return language_to_elevenlabs_language(language) return language_to_elevenlabs_language(language)
def _set_voice_settings(self): def _set_voice_settings(self):
return build_elevenlabs_voice_settings(self._settings) return build_elevenlabs_voice_settings(self._settings)
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the TTS model and reconnect.
Args:
model: The model name to use for synthesis.
"""
await super().set_model(model) await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]") logger.info(f"Switching TTS model to: [{model}]")
await self._disconnect() await self._disconnect()
await self._connect() await self._connect()
async def _update_settings(self, settings: Mapping[str, Any]): async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect if voice changed."""
prev_voice = self._voice_id prev_voice = self._voice_id
await super()._update_settings(settings) await super()._update_settings(settings)
if not prev_voice == self._voice_id: if not prev_voice == self._voice_id:
@@ -267,19 +349,35 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await self._connect() await self._connect()
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the ElevenLabs TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._output_format = output_format_from_sample_rate(self.sample_rate) self._output_format = output_format_from_sample_rate(self.sample_rate)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the ElevenLabs TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the ElevenLabs TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def flush_audio(self): async def flush_audio(self):
"""Flush any pending audio and finalize the current context."""
if not self._context_id or not self._websocket: if not self._context_id or not self._websocket:
return return
logger.trace(f"{self}: flushing audio") logger.trace(f"{self}: flushing audio")
@@ -287,6 +385,12 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
await self._websocket.send(json.dumps(msg)) await self._websocket.send(json.dumps(msg))
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame and handle state changes.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
await super().push_frame(frame, direction) await super().push_frame(frame, direction)
if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
self._started = False self._started = False
@@ -374,6 +478,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
raise Exception("Websocket not connected") raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
"""Handle interruption by closing the current context."""
await super()._handle_interruption(frame, direction) await super()._handle_interruption(frame, direction)
# Close the current context when interrupted without closing the websocket # Close the current context when interrupted without closing the websocket
@@ -395,6 +500,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
self._started = False self._started = False
async def _receive_messages(self): async def _receive_messages(self):
"""Handle incoming WebSocket messages from ElevenLabs."""
async for message in WatchdogAsyncIterator( async for message in WatchdogAsyncIterator(
self._get_websocket(), manager=self.task_manager self._get_websocket(), manager=self.task_manager
): ):
@@ -428,6 +534,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
self._cumulative_time = word_times[-1][1] self._cumulative_time = word_times[-1][1]
async def _keepalive_task_handler(self): async def _keepalive_task_handler(self):
"""Send periodic keepalive messages to maintain WebSocket connection."""
KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3 KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3
while True: while True:
self.reset_watchdog() self.reset_watchdog()
@@ -453,12 +560,21 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
break break
async def _send_text(self, text: str): async def _send_text(self, text: str):
"""Send text to the WebSocket for synthesis."""
if self._websocket and self._context_id: if self._websocket and self._context_id:
msg = {"text": text, "context_id": self._context_id} msg = {"text": text, "context_id": self._context_id}
await self._websocket.send(json.dumps(msg)) await self._websocket.send(json.dumps(msg))
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using ElevenLabs' streaming WebSocket API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:
@@ -497,19 +613,26 @@ class ElevenLabsTTSService(AudioContextWordTTSService):
class ElevenLabsHttpTTSService(WordTTSService): class ElevenLabsHttpTTSService(WordTTSService):
"""ElevenLabs Text-to-Speech service using HTTP streaming with word timestamps. """ElevenLabs HTTP-based TTS service with word timestamps.
Args: Provides text-to-speech using ElevenLabs' HTTP streaming API for simpler,
api_key: ElevenLabs API key non-WebSocket integration. Suitable for use cases where streaming WebSocket
voice_id: ID of the voice to use connection is not required or desired.
aiohttp_session: aiohttp ClientSession
model: Model ID (default: "eleven_flash_v2_5" for low latency)
base_url: API base URL
sample_rate: Output sample rate
params: Additional parameters for voice configuration
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for ElevenLabs HTTP TTS configuration.
Parameters:
language: Language to use for synthesis.
optimize_streaming_latency: Latency optimization level (0-4).
stability: Voice stability control (0.0 to 1.0).
similarity_boost: Similarity boost control (0.0 to 1.0).
style: Style control for voice expression (0.0 to 1.0).
use_speaker_boost: Whether to use speaker boost enhancement.
speed: Voice speed control (0.25 to 4.0).
"""
language: Optional[Language] = None language: Optional[Language] = None
optimize_streaming_latency: Optional[int] = None optimize_streaming_latency: Optional[int] = None
stability: Optional[float] = None stability: Optional[float] = None
@@ -530,6 +653,18 @@ class ElevenLabsHttpTTSService(WordTTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the ElevenLabs HTTP TTS service.
Args:
api_key: ElevenLabs API key for authentication.
voice_id: ID of the voice to use for synthesis.
aiohttp_session: aiohttp ClientSession for HTTP requests.
model: TTS model to use (e.g., "eleven_flash_v2_5").
base_url: Base URL for ElevenLabs HTTP API.
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent service.
"""
super().__init__( super().__init__(
aggregate_sentences=True, aggregate_sentences=True,
push_text_frames=False, push_text_frames=False,
@@ -569,11 +704,22 @@ class ElevenLabsHttpTTSService(WordTTSService):
self._previous_text = "" self._previous_text = ""
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language to ElevenLabs language code.""" """Convert pipecat Language to ElevenLabs language code.
Args:
language: The language to convert.
Returns:
The ElevenLabs-specific language code, or None if not supported.
"""
return language_to_elevenlabs_language(language) return language_to_elevenlabs_language(language)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Indicate that this service can generate usage metrics.""" """Check if this service can generate processing metrics.
Returns:
True, as ElevenLabs HTTP service supports metrics generation.
"""
return True return True
def _set_voice_settings(self): def _set_voice_settings(self):
@@ -587,12 +733,22 @@ class ElevenLabsHttpTTSService(WordTTSService):
logger.debug(f"{self}: Reset internal state") logger.debug(f"{self}: Reset internal state")
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Initialize the service upon receiving a StartFrame.""" """Start the ElevenLabs HTTP TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._output_format = output_format_from_sample_rate(self.sample_rate) self._output_format = output_format_from_sample_rate(self.sample_rate)
self._reset_state() self._reset_state()
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame and handle state changes.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
await super().push_frame(frame, direction) await super().push_frame(frame, direction)
if isinstance(frame, (StartInterruptionFrame, TTSStoppedFrame)): if isinstance(frame, (StartInterruptionFrame, TTSStoppedFrame)):
# Reset timing on interruption or stop # Reset timing on interruption or stop
@@ -619,10 +775,10 @@ class ElevenLabsHttpTTSService(WordTTSService):
[("Hello", 0.1), ("world", 0.5)] [("Hello", 0.1), ("world", 0.5)]
Args: Args:
alignment_info: Character timing data from ElevenLabs alignment_info: Character timing data from ElevenLabs.
Returns: Returns:
List of (word, timestamp) pairs List of (word, timestamp) pairs.
""" """
chars = alignment_info.get("characters", []) chars = alignment_info.get("characters", [])
char_start_times = alignment_info.get("character_start_times_seconds", []) char_start_times = alignment_info.get("character_start_times_seconds", [])
@@ -673,10 +829,10 @@ class ElevenLabsHttpTTSService(WordTTSService):
Includes previous text as context for better prosody continuity. Includes previous text as context for better prosody continuity.
Args: Args:
text: Text to convert to speech text: Text to convert to speech.
Yields: Yields:
Audio and control frames Frame: Audio and control frames containing the synthesized speech.
""" """
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Fal's image generation service implementation.
This module provides integration with Fal's image generation API
for creating images from text prompts using various AI models.
"""
import asyncio import asyncio
import io import io
import os import os
@@ -26,7 +32,25 @@ except ModuleNotFoundError as e:
class FalImageGenService(ImageGenService): class FalImageGenService(ImageGenService):
"""Fal's image generation service.
Provides text-to-image generation using Fal.ai's API with configurable
parameters for image quality, safety, and format options.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Fal.ai image generation.
Parameters:
seed: Random seed for reproducible generation. If None, uses random seed.
num_inference_steps: Number of inference steps for generation. Defaults to 8.
num_images: Number of images to generate. Defaults to 1.
image_size: Image dimensions as string preset or dict with width/height. Defaults to "square_hd".
expand_prompt: Whether to automatically expand/enhance the prompt. Defaults to False.
enable_safety_checker: Whether to enable content safety filtering. Defaults to True.
format: Output image format. Defaults to "png".
"""
seed: Optional[int] = None seed: Optional[int] = None
num_inference_steps: int = 8 num_inference_steps: int = 8
num_images: int = 1 num_images: int = 1
@@ -44,6 +68,15 @@ class FalImageGenService(ImageGenService):
key: Optional[str] = None, key: Optional[str] = None,
**kwargs, **kwargs,
): ):
"""Initialize the FalImageGenService.
Args:
params: Input parameters for image generation configuration.
aiohttp_session: HTTP client session for downloading generated images.
model: The Fal.ai model to use for generation. Defaults to "fal-ai/fast-sdxl".
key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable.
**kwargs: Additional arguments passed to parent ImageGenService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self.set_model_name(model) self.set_model_name(model)
self._params = params self._params = params
@@ -52,6 +85,16 @@ class FalImageGenService(ImageGenService):
os.environ["FAL_KEY"] = key os.environ["FAL_KEY"] = key
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
"""Generate an image from a text prompt.
Args:
prompt: The text prompt to generate an image from.
Yields:
URLImageRawFrame: Frame containing the generated image data and metadata.
ErrorFrame: If image generation fails.
"""
def load_image_bytes(encoded_image: bytes): def load_image_bytes(encoded_image: bytes):
buffer = io.BytesIO(encoded_image) buffer = io.BytesIO(encoded_image)
image = Image.open(buffer) image = Image.open(buffer)

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Fal speech-to-text service implementation.
This module provides integration with Fal's Wizper API for speech-to-text
transcription using segmented audio processing.
"""
import os import os
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
@@ -27,7 +33,14 @@ except ModuleNotFoundError as e:
def language_to_fal_language(language: Language) -> Optional[str]: def language_to_fal_language(language: Language) -> Optional[str]:
"""Language support for Fal's Wizper API.""" """Convert a Language enum to Fal's Wizper language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding Fal Wizper language code, or None if not supported.
"""
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.AF: "af", Language.AF: "af",
Language.AM: "am", Language.AM: "am",
@@ -145,18 +158,12 @@ class FalSTTService(SegmentedSTTService):
This service uses Fal's Wizper API to perform speech-to-text transcription on audio This service uses Fal's Wizper API to perform speech-to-text transcription on audio
segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection.
Args:
api_key: Fal API key. If not provided, will check FAL_KEY environment variable.
sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
params: Configuration parameters for the Wizper API.
**kwargs: Additional arguments passed to SegmentedSTTService.
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Configuration parameters for Fal's Wizper API. """Configuration parameters for Fal's Wizper API.
Attributes: Parameters:
language: Language of the audio input. Defaults to English. language: Language of the audio input. Defaults to English.
task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'. task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'.
chunk_level: Level of chunking ('segment'). Defaults to 'segment'. chunk_level: Level of chunking ('segment'). Defaults to 'segment'.
@@ -176,6 +183,14 @@ class FalSTTService(SegmentedSTTService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the FalSTTService with API key and parameters.
Args:
api_key: Fal API key. If not provided, will check FAL_KEY environment variable.
sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate.
params: Configuration parameters for the Wizper API.
**kwargs: Additional arguments passed to SegmentedSTTService.
"""
super().__init__( super().__init__(
sample_rate=sample_rate, sample_rate=sample_rate,
**kwargs, **kwargs,
@@ -201,16 +216,39 @@ class FalSTTService(SegmentedSTTService):
} }
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate processing metrics.
Returns:
True, as Fal STT service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Fal's service-specific language code.
Args:
language: The language to convert.
Returns:
The Fal-specific language code, or None if not supported.
"""
return language_to_fal_language(language) return language_to_fal_language(language)
async def set_language(self, language: Language): async def set_language(self, language: Language):
"""Set the transcription language.
Args:
language: The language to use for speech-to-text transcription.
"""
logger.info(f"Switching STT language to: [{language}]") logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = self.language_to_service_language(language) self._settings["language"] = self.language_to_service_language(language)
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the STT model.
Args:
model: The model name to use for transcription.
"""
await super().set_model(model) await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]") logger.info(f"Switching STT model to: [{model}]")
@@ -229,7 +267,7 @@ class FalSTTService(SegmentedSTTService):
audio: Raw audio bytes in WAV format (already converted by base class). audio: Raw audio bytes in WAV format (already converted by base class).
Yields: Yields:
Frame: TranscriptionFrame containing the transcribed text. Frame: TranscriptionFrame containing the transcribed text, or ErrorFrame on failure.
Note: Note:
The audio is already in WAV format from the SegmentedSTTService. The audio is already in WAV format from the SegmentedSTTService.

View File

@@ -20,12 +20,6 @@ class FireworksLLMService(OpenAILLMService):
This service extends OpenAILLMService to connect to Fireworks' API endpoint while This service extends OpenAILLMService to connect to Fireworks' API endpoint while
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Args:
api_key: The API key for accessing Fireworks AI.
model: The model identifier to use. Defaults to "accounts/fireworks/models/firefunction-v2".
base_url: The base URL for Fireworks API. Defaults to "https://api.fireworks.ai/inference/v1".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -36,6 +30,14 @@ class FireworksLLMService(OpenAILLMService):
base_url: str = "https://api.fireworks.ai/inference/v1", base_url: str = "https://api.fireworks.ai/inference/v1",
**kwargs, **kwargs,
): ):
"""Initialize the Fireworks LLM service.
Args:
api_key: The API key for accessing Fireworks AI.
model: The model identifier to use. Defaults to "accounts/fireworks/models/firefunction-v2".
base_url: The base URL for Fireworks API. Defaults to "https://api.fireworks.ai/inference/v1".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
def create_client(self, api_key=None, base_url=None, **kwargs): def create_client(self, api_key=None, base_url=None, **kwargs):

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Fish Audio text-to-speech service implementation.
This module provides integration with Fish Audio's real-time TTS WebSocket API
for streaming text-to-speech synthesis with customizable voice parameters.
"""
import uuid import uuid
from typing import AsyncGenerator, Literal, Optional from typing import AsyncGenerator, Literal, Optional
@@ -39,7 +45,23 @@ FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"]
class FishAudioTTSService(InterruptibleTTSService): class FishAudioTTSService(InterruptibleTTSService):
"""Fish Audio text-to-speech service with WebSocket streaming.
Provides real-time text-to-speech synthesis using Fish Audio's WebSocket API.
Supports various audio formats, customizable prosody controls, and streaming
audio generation with interruption handling.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Fish Audio TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
latency: Latency mode ("normal" or "balanced"). Defaults to "normal".
prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0.
prosody_volume: Volume adjustment in dB. Defaults to 0.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
latency: Optional[str] = "normal" # "normal" or "balanced" latency: Optional[str] = "normal" # "normal" or "balanced"
prosody_speed: Optional[float] = 1.0 # Speech speed (0.5-2.0) prosody_speed: Optional[float] = 1.0 # Speech speed (0.5-2.0)
@@ -55,6 +77,16 @@ class FishAudioTTSService(InterruptibleTTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Fish Audio TTS service.
Args:
api_key: Fish Audio API key for authentication.
model: Reference ID of the voice model to use for synthesis.
output_format: Audio output format. Defaults to "pcm".
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent service.
"""
super().__init__( super().__init__(
push_stop_frames=True, push_stop_frames=True,
pause_frame_processing=True, pause_frame_processing=True,
@@ -85,23 +117,48 @@ class FishAudioTTSService(InterruptibleTTSService):
self.set_model_name(model) self.set_model_name(model)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Fish Audio service supports metrics generation.
"""
return True return True
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the TTS model (reference ID).
Args:
model: The reference ID of the voice model to use.
"""
self._settings["reference_id"] = model self._settings["reference_id"] = model
await super().set_model(model) await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]") logger.info(f"Switching TTS model to: [{model}]")
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Fish Audio TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._settings["sample_rate"] = self.sample_rate self._settings["sample_rate"] = self.sample_rate
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the Fish Audio TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the Fish Audio TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
@@ -191,6 +248,14 @@ class FishAudioTTSService(InterruptibleTTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Fish Audio's streaming API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames and control frames for the synthesized speech.
"""
logger.debug(f"{self}: Generating Fish TTS: [{text}]") logger.debug(f"{self}: Generating Fish TTS: [{text}]")
try: try:
if not self._websocket or self._websocket.closed: if not self._websocket or self._websocket.closed:

View File

@@ -333,7 +333,12 @@ class GeminiMultimodalLiveContextAggregatorPair:
class GeminiMultimodalModalities(Enum): class GeminiMultimodalModalities(Enum):
"""Supported modalities for Gemini Multimodal Live.""" """Supported modalities for Gemini Multimodal Live.
Parameters:
TEXT: Text responses.
AUDIO: Audio responses.
"""
TEXT = "TEXT" TEXT = "TEXT"
AUDIO = "AUDIO" AUDIO = "AUDIO"
@@ -422,20 +427,6 @@ class GeminiMultimodalLiveLLMService(LLMService):
This service enables real-time conversations with Gemini, supporting both This service enables real-time conversations with Gemini, supporting both
text and audio modalities. It handles voice transcription, streaming audio text and audio modalities. It handles voice transcription, streaming audio
responses, and tool usage. responses, and tool usage.
Args:
api_key: Google AI API key for authentication.
base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint.
model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001".
voice_id: TTS voice identifier. Defaults to "Charon".
start_audio_paused: Whether to start with audio input paused. Defaults to False.
start_video_paused: Whether to start with video input paused. Defaults to False.
system_instruction: System prompt for the model. Defaults to None.
tools: Tools/functions available to the model. Defaults to None.
params: Configuration parameters for the model. Defaults to InputParams().
inference_on_context_initialization: Whether to generate a response when context
is first set. Defaults to True.
**kwargs: Additional arguments passed to parent LLMService.
""" """
# Overriding the default adapter to use the Gemini one. # Overriding the default adapter to use the Gemini one.
@@ -456,6 +447,22 @@ class GeminiMultimodalLiveLLMService(LLMService):
inference_on_context_initialization: bool = True, inference_on_context_initialization: bool = True,
**kwargs, **kwargs,
): ):
"""Initialize the Gemini Multimodal Live LLM service.
Args:
api_key: Google AI API key for authentication.
base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint.
model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001".
voice_id: TTS voice identifier. Defaults to "Charon".
start_audio_paused: Whether to start with audio input paused. Defaults to False.
start_video_paused: Whether to start with video input paused. Defaults to False.
system_instruction: System prompt for the model. Defaults to None.
tools: Tools/functions available to the model. Defaults to None.
params: Configuration parameters for the model. Defaults to InputParams().
inference_on_context_initialization: Whether to generate a response when context
is first set. Defaults to True.
**kwargs: Additional arguments passed to parent LLMService.
"""
super().__init__(base_url=base_url, **kwargs) super().__init__(base_url=base_url, **kwargs)
params = params or InputParams() params = params or InputParams()

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Configuration for the Gladia STT service."""
from typing import Any, Dict, List, Optional, Union from typing import Any, Dict, List, Optional, Union
from pydantic import BaseModel from pydantic import BaseModel
@@ -14,7 +16,7 @@ from pipecat.transcriptions.language import Language
class LanguageConfig(BaseModel): class LanguageConfig(BaseModel):
"""Configuration for language detection and handling. """Configuration for language detection and handling.
Attributes: Parameters:
languages: List of language codes to use for transcription languages: List of language codes to use for transcription
code_switching: Whether to auto-detect language changes during transcription code_switching: Whether to auto-detect language changes during transcription
""" """
@@ -26,7 +28,7 @@ class LanguageConfig(BaseModel):
class PreProcessingConfig(BaseModel): class PreProcessingConfig(BaseModel):
"""Configuration for audio pre-processing options. """Configuration for audio pre-processing options.
Attributes: Parameters:
speech_threshold: Sensitivity for speech detection (0-1) speech_threshold: Sensitivity for speech detection (0-1)
""" """
@@ -36,7 +38,7 @@ class PreProcessingConfig(BaseModel):
class CustomVocabularyItem(BaseModel): class CustomVocabularyItem(BaseModel):
"""Represents a custom vocabulary item with an intensity value. """Represents a custom vocabulary item with an intensity value.
Attributes: Parameters:
value: The vocabulary word or phrase value: The vocabulary word or phrase
intensity: The bias intensity for this vocabulary item (0-1) intensity: The bias intensity for this vocabulary item (0-1)
""" """
@@ -48,7 +50,7 @@ class CustomVocabularyItem(BaseModel):
class CustomVocabularyConfig(BaseModel): class CustomVocabularyConfig(BaseModel):
"""Configuration for custom vocabulary. """Configuration for custom vocabulary.
Attributes: Parameters:
vocabulary: List of words/phrases or CustomVocabularyItem objects vocabulary: List of words/phrases or CustomVocabularyItem objects
default_intensity: Default intensity for simple string vocabulary items default_intensity: Default intensity for simple string vocabulary items
""" """
@@ -60,7 +62,7 @@ class CustomVocabularyConfig(BaseModel):
class CustomSpellingConfig(BaseModel): class CustomSpellingConfig(BaseModel):
"""Configuration for custom spelling rules. """Configuration for custom spelling rules.
Attributes: Parameters:
spelling_dictionary: Mapping of correct spellings to phonetic variations spelling_dictionary: Mapping of correct spellings to phonetic variations
""" """
@@ -70,7 +72,7 @@ class CustomSpellingConfig(BaseModel):
class TranslationConfig(BaseModel): class TranslationConfig(BaseModel):
"""Configuration for real-time translation. """Configuration for real-time translation.
Attributes: Parameters:
target_languages: List of target language codes for translation target_languages: List of target language codes for translation
model: Translation model to use ("base" or "enhanced") model: Translation model to use ("base" or "enhanced")
match_original_utterances: Whether to align translations with original utterances match_original_utterances: Whether to align translations with original utterances
@@ -92,7 +94,7 @@ class TranslationConfig(BaseModel):
class RealtimeProcessingConfig(BaseModel): class RealtimeProcessingConfig(BaseModel):
"""Configuration for real-time processing features. """Configuration for real-time processing features.
Attributes: Parameters:
words_accurate_timestamps: Whether to provide per-word timestamps words_accurate_timestamps: Whether to provide per-word timestamps
custom_vocabulary: Whether to enable custom vocabulary custom_vocabulary: Whether to enable custom vocabulary
custom_vocabulary_config: Custom vocabulary configuration custom_vocabulary_config: Custom vocabulary configuration
@@ -118,7 +120,7 @@ class RealtimeProcessingConfig(BaseModel):
class MessagesConfig(BaseModel): class MessagesConfig(BaseModel):
"""Configuration for controlling which message types are sent via WebSocket. """Configuration for controlling which message types are sent via WebSocket.
Attributes: Parameters:
receive_partial_transcripts: Whether to receive intermediate transcription results receive_partial_transcripts: Whether to receive intermediate transcription results
receive_final_transcripts: Whether to receive final transcription results receive_final_transcripts: Whether to receive final transcription results
receive_speech_events: Whether to receive speech begin/end events receive_speech_events: Whether to receive speech begin/end events
@@ -144,7 +146,7 @@ class MessagesConfig(BaseModel):
class GladiaInputParams(BaseModel): class GladiaInputParams(BaseModel):
"""Configuration parameters for the Gladia STT service. """Configuration parameters for the Gladia STT service.
Attributes: Parameters:
encoding: Audio encoding format encoding: Audio encoding format
bit_depth: Audio bit depth bit_depth: Audio bit depth
channels: Number of audio channels channels: Number of audio channels

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Gladia Speech-to-Text (STT) service implementation.
This module provides a Speech-to-Text service using Gladia's real-time WebSocket API,
supporting multiple languages, custom vocabulary, and various audio processing options.
"""
import asyncio import asyncio
import base64 import base64
import json import json
@@ -41,10 +47,10 @@ def language_to_gladia_language(language: Language) -> Optional[str]:
"""Convert a Language enum to Gladia's language code format. """Convert a Language enum to Gladia's language code format.
Args: Args:
language: The Language enum value to convert language: The Language enum value to convert.
Returns: Returns:
The Gladia language code string or None if not supported The Gladia language code string or None if not supported.
""" """
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.AF: "af", Language.AF: "af",
@@ -180,6 +186,7 @@ class GladiaSTTService(STTService):
This service connects to Gladia's WebSocket API for real-time transcription This service connects to Gladia's WebSocket API for real-time transcription
with support for multiple languages, custom vocabulary, and various processing options. with support for multiple languages, custom vocabulary, and various processing options.
Provides automatic reconnection, audio buffering, and comprehensive error handling.
For complete API documentation, see: https://docs.gladia.io/api-reference/v2/live/init For complete API documentation, see: https://docs.gladia.io/api-reference/v2/live/init
""" """
@@ -204,16 +211,16 @@ class GladiaSTTService(STTService):
"""Initialize the Gladia STT service. """Initialize the Gladia STT service.
Args: Args:
api_key: Gladia API key api_key: Gladia API key for authentication.
url: Gladia API URL url: Gladia API URL. Defaults to "https://api.gladia.io/v2/live".
confidence: Minimum confidence threshold for transcriptions confidence: Minimum confidence threshold for transcriptions (0.0-1.0).
sample_rate: Audio sample rate in Hz sample_rate: Audio sample rate in Hz. If None, uses service default.
model: Model to use ("solaria-1") model: Model to use for transcription. Defaults to "solaria-1".
params: Additional configuration parameters params: Additional configuration parameters for Gladia service.
max_reconnection_attempts: Maximum number of reconnection attempts max_reconnection_attempts: Maximum number of reconnection attempts. Defaults to 5.
reconnection_delay: Initial delay between reconnection attempts (exponential backoff) reconnection_delay: Initial delay between reconnection attempts in seconds.
max_buffer_size: Maximum size of audio buffer in bytes max_buffer_size: Maximum size of audio buffer in bytes. Defaults to 20MB.
**kwargs: Additional arguments passed to the STTService **kwargs: Additional arguments passed to the STTService parent class.
""" """
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
@@ -256,10 +263,22 @@ class GladiaSTTService(STTService):
self._should_reconnect = True self._should_reconnect = True
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate performance metrics.
Returns:
True, indicating this service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert pipecat Language enum to Gladia's language code.""" """Convert pipecat Language enum to Gladia's language code.
Args:
language: The Language enum value to convert.
Returns:
The Gladia language code string or None if not supported.
"""
return language_to_gladia_language(language) return language_to_gladia_language(language)
def _prepare_settings(self) -> Dict[str, Any]: def _prepare_settings(self) -> Dict[str, Any]:
@@ -314,7 +333,11 @@ class GladiaSTTService(STTService):
return settings return settings
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Gladia STT websocket connection.""" """Start the Gladia STT websocket connection.
Args:
frame: The start frame triggering service startup.
"""
await super().start(frame) await super().start(frame)
if self._connection_task: if self._connection_task:
return return
@@ -323,7 +346,11 @@ class GladiaSTTService(STTService):
self._connection_task = self.create_task(self._connection_handler()) self._connection_task = self.create_task(self._connection_handler())
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the Gladia STT websocket connection.""" """Stop the Gladia STT websocket connection.
Args:
frame: The end frame triggering service shutdown.
"""
await super().stop(frame) await super().stop(frame)
self._should_reconnect = False self._should_reconnect = False
await self._send_stop_recording() await self._send_stop_recording()
@@ -335,7 +362,11 @@ class GladiaSTTService(STTService):
await self._cleanup_connection() await self._cleanup_connection()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the Gladia STT websocket connection.""" """Cancel the Gladia STT websocket connection.
Args:
frame: The cancel frame triggering service cancellation.
"""
await super().cancel(frame) await super().cancel(frame)
self._should_reconnect = False self._should_reconnect = False
@@ -346,7 +377,14 @@ class GladiaSTTService(STTService):
await self._cleanup_connection() await self._cleanup_connection()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Run speech-to-text on audio data.""" """Run speech-to-text on audio data.
Args:
audio: Raw audio bytes to transcribe.
Yields:
None (processing is handled asynchronously via WebSocket).
"""
await self.start_ttfb_metrics() await self.start_ttfb_metrics()
await self.start_processing_metrics() await self.start_processing_metrics()

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Google AI service frames for search and grounding functionality.
This module defines specialized frame types for handling search results
and grounding metadata from Google AI models, particularly for Gemini
models that support web search and fact grounding capabilities.
"""
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import List, Optional from typing import List, Optional
@@ -12,12 +19,27 @@ from pipecat.frames.frames import DataFrame
@dataclass @dataclass
class LLMSearchResult: class LLMSearchResult:
"""Represents a single search result with confidence scores.
Parameters:
text: The search result text content.
confidence: List of confidence scores associated with the result.
"""
text: str text: str
confidence: List[float] = field(default_factory=list) confidence: List[float] = field(default_factory=list)
@dataclass @dataclass
class LLMSearchOrigin: class LLMSearchOrigin:
"""Represents the origin source of search results.
Parameters:
site_uri: URI of the source website.
site_title: Title of the source website.
results: List of search results from this origin.
"""
site_uri: Optional[str] = None site_uri: Optional[str] = None
site_title: Optional[str] = None site_title: Optional[str] = None
results: List[LLMSearchResult] = field(default_factory=list) results: List[LLMSearchResult] = field(default_factory=list)
@@ -25,9 +47,27 @@ class LLMSearchOrigin:
@dataclass @dataclass
class LLMSearchResponseFrame(DataFrame): class LLMSearchResponseFrame(DataFrame):
"""Frame containing search results and grounding information from Google AI models.
This frame is used to convey search results and grounding metadata
from Google AI models that support web search capabilities. It includes
the search result text, rendered content, and detailed origin information
with confidence scores.
Parameters:
search_result: The main search result text.
rendered_content: Rendered content from the search entry point.
origins: List of search result origins with detailed information.
"""
search_result: Optional[str] = None search_result: Optional[str] = None
rendered_content: Optional[str] = None rendered_content: Optional[str] = None
origins: List[LLMSearchOrigin] = field(default_factory=list) origins: List[LLMSearchOrigin] = field(default_factory=list)
def __str__(self): def __str__(self):
"""Return string representation of the search response frame.
Returns:
String representation showing search result and origins.
"""
return f"LLMSearchResponseFrame(search_result={self.search_result}, origins={self.origins})" return f"LLMSearchResponseFrame(search_result={self.search_result}, origins={self.origins})"

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Google AI image generation service implementation.
This module provides integration with Google's Imagen model for generating
images from text prompts using the Google AI API.
"""
import io import io
import os import os
@@ -29,7 +35,22 @@ except ModuleNotFoundError as e:
class GoogleImageGenService(ImageGenService): class GoogleImageGenService(ImageGenService):
"""Google AI image generation service using Imagen models.
Provides text-to-image generation capabilities using Google's Imagen models
through the Google AI API. Supports multiple image generation and negative
prompting for enhanced control over generated content.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Configuration parameters for Google image generation.
Parameters:
number_of_images: Number of images to generate (1-8). Defaults to 1.
model: Google Imagen model to use. Defaults to "imagen-3.0-generate-002".
negative_prompt: Optional negative prompt to guide what not to include.
"""
number_of_images: int = Field(default=1, ge=1, le=8) number_of_images: int = Field(default=1, ge=1, le=8)
model: str = Field(default="imagen-3.0-generate-002") model: str = Field(default="imagen-3.0-generate-002")
negative_prompt: Optional[str] = Field(default=None) negative_prompt: Optional[str] = Field(default=None)
@@ -41,22 +62,38 @@ class GoogleImageGenService(ImageGenService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the GoogleImageGenService with API key and parameters.
Args:
api_key: Google AI API key for authentication.
params: Configuration parameters for image generation. Defaults to InputParams().
**kwargs: Additional arguments passed to the parent ImageGenService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._params = params or GoogleImageGenService.InputParams() self._params = params or GoogleImageGenService.InputParams()
self._client = genai.Client(api_key=api_key) self._client = genai.Client(api_key=api_key)
self.set_model_name(self._params.model) self.set_model_name(self._params.model)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Google image generation service supports metrics.
"""
return True return True
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
"""Generate images from a text prompt using Google's Imagen model. """Generate images from a text prompt using Google's Imagen model.
Args: Args:
prompt (str): The text description to generate images from. prompt: The text description to generate images from.
Yields: Yields:
Frame: Generated image frames or error frames. Frame: Generated URLImageRawFrame objects containing the generated
images, or ErrorFrame objects if generation fails.
Raises:
Exception: If there are issues with the Google AI API or image processing.
""" """
logger.debug(f"Generating image from prompt: {prompt}") logger.debug(f"Generating image from prompt: {prompt}")
await self.start_ttfb_metrics() await self.start_ttfb_metrics()

View File

@@ -233,11 +233,6 @@ class GoogleLLMContext(OpenAILLMContext):
This class handles conversion between OpenAI-style messages and Google AI's This class handles conversion between OpenAI-style messages and Google AI's
Content/Part format, including system messages, function calls, and media. Content/Part format, including system messages, function calls, and media.
Args:
messages: Initial messages in OpenAI format.
tools: Available tools/functions for the model.
tool_choice: Tool choice configuration.
""" """
def __init__( def __init__(
@@ -246,6 +241,13 @@ class GoogleLLMContext(OpenAILLMContext):
tools: Optional[List[dict]] = None, tools: Optional[List[dict]] = None,
tool_choice: Optional[dict] = None, tool_choice: Optional[dict] = None,
): ):
"""Initialize GoogleLLMContext.
Args:
messages: Initial messages in OpenAI format.
tools: Available tools/functions for the model.
tool_choice: Tool choice configuration.
"""
super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) super().__init__(messages=messages, tools=tools, tool_choice=tool_choice)
self.system_message = None self.system_message = None
@@ -563,15 +565,6 @@ class GoogleLLMService(LLMService):
from OpenAILLMContext to the messages format expected by the Google AI model. from OpenAILLMContext to the messages format expected by the Google AI model.
We use OpenAILLMContext as a lingua franca for all LLM services to enable We use OpenAILLMContext as a lingua franca for all LLM services to enable
easy switching between different LLMs. easy switching between different LLMs.
Args:
api_key: Google AI API key for authentication.
model: Model name to use. Defaults to "gemini-2.0-flash".
params: Input parameters for the model.
system_instruction: System instruction/prompt for the model.
tools: List of available tools/functions.
tool_config: Configuration for tool usage.
**kwargs: Additional arguments passed to parent class.
""" """
# Overriding the default adapter to use the Gemini one. # Overriding the default adapter to use the Gemini one.
@@ -605,6 +598,17 @@ class GoogleLLMService(LLMService):
tool_config: Optional[Dict[str, Any]] = None, tool_config: Optional[Dict[str, Any]] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Google LLM service.
Args:
api_key: Google AI API key for authentication.
model: Model name to use. Defaults to "gemini-2.0-flash".
params: Input parameters for the model.
system_instruction: System instruction/prompt for the model.
tools: List of available tools/functions.
tool_config: Configuration for tool usage.
**kwargs: Additional arguments passed to parent class.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
params = params or GoogleLLMService.InputParams() params = params or GoogleLLMService.InputParams()

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Google LLM service using OpenAI-compatible API format.
This module provides integration with Google's AI LLM models using the OpenAI
API format through Google's Gemini API OpenAI compatibility layer.
"""
import json import json
import os import os
@@ -25,8 +31,17 @@ from pipecat.services.openai.llm import OpenAILLMService
class GoogleLLMOpenAIBetaService(OpenAILLMService): class GoogleLLMOpenAIBetaService(OpenAILLMService):
"""This class implements inference with Google's AI LLM models using the OpenAI format. """Google LLM service using OpenAI-compatible API format.
Ref - https://ai.google.dev/gemini-api/docs/openai
This service provides access to Google's AI LLM models (like Gemini) through
the OpenAI API format. It handles streaming responses, function calls, and
tool usage while maintaining compatibility with OpenAI's interface.
Note: This service includes a workaround for a Google API bug where function
call indices may be incorrectly set to None, resulting in empty function names.
Reference:
https://ai.google.dev/gemini-api/docs/openai
""" """
def __init__( def __init__(
@@ -37,6 +52,14 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService):
model: str = "gemini-2.0-flash", model: str = "gemini-2.0-flash",
**kwargs, **kwargs,
): ):
"""Initialize the Google LLM service.
Args:
api_key: Google API key for authentication.
base_url: Base URL for Google's OpenAI-compatible API.
model: Google model name to use (e.g., "gemini-2.0-flash").
**kwargs: Additional arguments passed to the parent OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
async def _process_context(self, context: OpenAILLMContext): async def _process_context(self, context: OpenAILLMContext):

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Google Vertex AI LLM service implementation.
This module provides integration with Google's AI models via Vertex AI while
maintaining OpenAI API compatibility through Google's OpenAI-compatible endpoint.
"""
import json import json
import os import os
@@ -31,16 +37,24 @@ except ModuleNotFoundError as e:
class GoogleVertexLLMService(OpenAILLMService): class GoogleVertexLLMService(OpenAILLMService):
"""Implements inference with Google's AI models via Vertex AI while """Google Vertex AI LLM service with OpenAI API compatibility.
maintaining OpenAI API compatibility.
Provides access to Google's AI models via Vertex AI while maintaining
OpenAI API compatibility. Handles authentication using Google service
account credentials and constructs appropriate endpoint URLs for
different GCP regions and projects.
Reference: Reference:
https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library
""" """
class InputParams(OpenAILLMService.InputParams): class InputParams(OpenAILLMService.InputParams):
"""Input parameters specific to Vertex AI.""" """Input parameters specific to Vertex AI.
Parameters:
location: GCP region for Vertex AI endpoint (e.g., "us-east4").
project_id: Google Cloud project ID.
"""
# https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations
location: str = "us-east4" location: str = "us-east4"
@@ -58,11 +72,11 @@ class GoogleVertexLLMService(OpenAILLMService):
"""Initializes the VertexLLMService. """Initializes the VertexLLMService.
Args: Args:
credentials (Optional[str]): JSON string of service account credentials. credentials: JSON string of service account credentials.
credentials_path (Optional[str]): Path to the service account JSON file. credentials_path: Path to the service account JSON file.
model (str): Model identifier. Defaults to "google/gemini-2.0-flash-001". model: Model identifier (e.g., "google/gemini-2.0-flash-001").
params (InputParams): Vertex AI input parameters. params: Vertex AI input parameters including location and project.
**kwargs: Additional arguments for OpenAILLMService. **kwargs: Additional arguments passed to OpenAILLMService.
""" """
params = params or OpenAILLMService.InputParams() params = params or OpenAILLMService.InputParams()
base_url = self._get_base_url(params) base_url = self._get_base_url(params)
@@ -74,7 +88,7 @@ class GoogleVertexLLMService(OpenAILLMService):
@staticmethod @staticmethod
def _get_base_url(params: InputParams) -> str: def _get_base_url(params: InputParams) -> str:
"""Constructs the base URL for Vertex AI API.""" """Construct the base URL for Vertex AI API."""
return ( return (
f"https://{params.location}-aiplatform.googleapis.com/v1/" f"https://{params.location}-aiplatform.googleapis.com/v1/"
f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi"
@@ -82,14 +96,22 @@ class GoogleVertexLLMService(OpenAILLMService):
@staticmethod @staticmethod
def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str:
"""Retrieves an authentication token using Google service account credentials. """Retrieve an authentication token using Google service account credentials.
Supports multiple authentication methods:
1. Direct JSON credentials string
2. Path to service account JSON file
3. Default application credentials (ADC)
Args: Args:
credentials (Optional[str]): JSON string of service account credentials. credentials: JSON string of service account credentials.
credentials_path (Optional[str]): Path to the service account JSON file. credentials_path: Path to the service account JSON file.
Returns: Returns:
str: OAuth token for API authentication. OAuth token for API authentication.
Raises:
ValueError: If no valid credentials are provided or found.
""" """
creds: Optional[service_account.Credentials] = None creds: Optional[service_account.Credentials] = None

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Google RTVI integration models and observer implementation.
This module provides integration with Google's services through the RTVI framework,
including models for search responses and an observer for handling Google-specific
frame types.
"""
from typing import List, Literal, Optional from typing import List, Literal, Optional
from pydantic import BaseModel from pydantic import BaseModel
@@ -16,22 +23,56 @@ from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFra
class RTVISearchResponseMessageData(BaseModel): class RTVISearchResponseMessageData(BaseModel):
"""Data payload for search response messages in RTVI protocol.
Parameters:
search_result: The search result text, if available.
rendered_content: The rendered content from the search, if available.
origins: List of search result origins with metadata.
"""
search_result: Optional[str] search_result: Optional[str]
rendered_content: Optional[str] rendered_content: Optional[str]
origins: List[LLMSearchOrigin] origins: List[LLMSearchOrigin]
class RTVIBotLLMSearchResponseMessage(BaseModel): class RTVIBotLLMSearchResponseMessage(BaseModel):
"""RTVI message for bot LLM search responses.
Parameters:
label: Always "rtvi-ai" for RTVI protocol messages.
type: Always "bot-llm-search-response" for this message type.
data: The search response data payload.
"""
label: Literal["rtvi-ai"] = "rtvi-ai" label: Literal["rtvi-ai"] = "rtvi-ai"
type: Literal["bot-llm-search-response"] = "bot-llm-search-response" type: Literal["bot-llm-search-response"] = "bot-llm-search-response"
data: RTVISearchResponseMessageData data: RTVISearchResponseMessageData
class GoogleRTVIObserver(RTVIObserver): class GoogleRTVIObserver(RTVIObserver):
"""RTVI observer for Google service integration.
Extends the base RTVIObserver to handle Google-specific frame types,
particularly LLM search response frames from Google services.
"""
def __init__(self, rtvi: RTVIProcessor): def __init__(self, rtvi: RTVIProcessor):
"""Initialize the Google RTVI observer.
Args:
rtvi: The RTVI processor to send messages through.
"""
super().__init__(rtvi) super().__init__(rtvi)
async def on_push_frame(self, data: FramePushed): async def on_push_frame(self, data: FramePushed):
"""Process frames being pushed through the pipeline.
Handles Google-specific frames in addition to the base RTVI frame types.
Args:
data: Frame push event data containing frame and metadata.
"""
await super().on_push_frame(data) await super().on_push_frame(data)
frame = data.frame frame = data.frame

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Google Cloud Speech-to-Text V2 service implementation for Pipecat.
This module provides a Google Cloud Speech-to-Text V2 service with streaming
support, enabling real-time speech recognition with features like automatic
punctuation, voice activity detection, and multi-language support.
"""
import asyncio import asyncio
import json import json
import os import os
@@ -353,9 +360,15 @@ class GoogleSTTService(STTService):
Provides real-time speech recognition using Google Cloud's Speech-to-Text V2 API Provides real-time speech recognition using Google Cloud's Speech-to-Text V2 API
with streaming support. Handles audio transcription and optional voice activity detection. with streaming support. Handles audio transcription and optional voice activity detection.
Implements automatic stream reconnection to handle Google's 4-minute streaming limit.
Attributes: Attributes:
InputParams: Configuration parameters for the STT service. InputParams: Configuration parameters for the STT service.
STREAMING_LIMIT: Google Cloud's streaming limit in milliseconds (4 minutes).
Raises:
ValueError: If neither credentials nor credentials_path is provided.
ValueError: If project ID is not found in credentials.
""" """
# Google Cloud's STT service has a connection time limit of 5 minutes per stream. # Google Cloud's STT service has a connection time limit of 5 minutes per stream.
@@ -367,7 +380,7 @@ class GoogleSTTService(STTService):
class InputParams(BaseModel): class InputParams(BaseModel):
"""Configuration parameters for Google Speech-to-Text. """Configuration parameters for Google Speech-to-Text.
Attributes: Parameters:
languages: Single language or list of recognition languages. First language is primary. languages: Single language or list of recognition languages. First language is primary.
model: Speech recognition model to use. model: Speech recognition model to use.
use_separate_recognition_per_channel: Process each audio channel separately. use_separate_recognition_per_channel: Process each audio channel separately.
@@ -396,13 +409,25 @@ class GoogleSTTService(STTService):
@field_validator("languages", mode="before") @field_validator("languages", mode="before")
@classmethod @classmethod
def validate_languages(cls, v) -> List[Language]: def validate_languages(cls, v) -> List[Language]:
"""Ensure languages is always a list.
Args:
v: Single Language enum or list of Language enums.
Returns:
List[Language]: List of configured languages.
"""
if isinstance(v, Language): if isinstance(v, Language):
return [v] return [v]
return v return v
@property @property
def language_list(self) -> List[Language]: def language_list(self) -> List[Language]:
"""Get languages as a guaranteed list.""" """Get languages as a guaranteed list.
Returns:
List[Language]: List of configured languages.
"""
assert isinstance(self.languages, list) assert isinstance(self.languages, list)
return self.languages return self.languages
@@ -425,10 +450,6 @@ class GoogleSTTService(STTService):
sample_rate: Audio sample rate in Hertz. sample_rate: Audio sample rate in Hertz.
params: Configuration parameters for the service. params: Configuration parameters for the service.
**kwargs: Additional arguments passed to STTService. **kwargs: Additional arguments passed to STTService.
Raises:
ValueError: If neither credentials nor credentials_path is provided.
ValueError: If project ID is not found in credentials.
""" """
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
@@ -501,6 +522,11 @@ class GoogleSTTService(STTService):
} }
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if the service can generate metrics.
Returns:
bool: True, as this service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language | List[Language]) -> str | List[str]: def language_to_service_language(self, language: Language | List[Language]) -> str | List[str]:
@@ -548,7 +574,11 @@ class GoogleSTTService(STTService):
await self._reconnect_if_needed() await self._reconnect_if_needed()
async def set_model(self, model: str): async def set_model(self, model: str):
"""Update the service's recognition model.""" """Update the service's recognition model.
Args:
model: The new recognition model to use.
"""
logger.debug(f"Switching STT model to: {model}") logger.debug(f"Switching STT model to: {model}")
await super().set_model(model) await super().set_model(model)
self._settings["model"] = model self._settings["model"] = model
@@ -556,14 +586,29 @@ class GoogleSTTService(STTService):
await self._reconnect_if_needed() await self._reconnect_if_needed()
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the STT service and establish connection.
Args:
frame: The start frame triggering the service start.
"""
await super().start(frame) await super().start(frame)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the STT service and clean up resources.
Args:
frame: The end frame triggering the service stop.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the STT service and clean up resources.
Args:
frame: The cancel frame triggering the service cancellation.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
@@ -585,7 +630,7 @@ class GoogleSTTService(STTService):
"""Update service options dynamically. """Update service options dynamically.
Args: Args:
languages: New list of recongition languages. languages: New list of recognition languages.
model: New recognition model. model: New recognition model.
enable_automatic_punctuation: Enable/disable automatic punctuation. enable_automatic_punctuation: Enable/disable automatic punctuation.
enable_spoken_punctuation: Enable/disable spoken punctuation. enable_spoken_punctuation: Enable/disable spoken punctuation.
@@ -767,7 +812,14 @@ class GoogleSTTService(STTService):
await self.push_frame(ErrorFrame(str(e))) await self.push_frame(ErrorFrame(str(e)))
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Process an audio chunk for STT transcription.""" """Process an audio chunk for STT transcription.
Args:
audio: Raw audio bytes to transcribe.
Yields:
Frame: None (actual transcription frames are pushed via internal processing).
"""
if self._streaming_task: if self._streaming_task:
# Queue the audio data # Queue the audio data
await self.start_ttfb_metrics() await self.start_ttfb_metrics()

View File

@@ -4,7 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
import asyncio """Google Cloud Text-to-Speech service implementations.
This module provides integration with Google Cloud Text-to-Speech API,
offering both HTTP-based synthesis with SSML support and streaming synthesis
for real-time applications.
"""
import json import json
import os import os
@@ -43,6 +49,14 @@ except ModuleNotFoundError as e:
def language_to_google_tts_language(language: Language) -> Optional[str]: def language_to_google_tts_language(language: Language) -> Optional[str]:
"""Convert a Language enum to Google TTS language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding Google TTS language code, or None if not supported.
"""
language_map = { language_map = {
# Afrikaans # Afrikaans
Language.AF: "af-ZA", Language.AF: "af-ZA",
@@ -203,7 +217,32 @@ def language_to_google_tts_language(language: Language) -> Optional[str]:
class GoogleHttpTTSService(TTSService): class GoogleHttpTTSService(TTSService):
"""Google Cloud Text-to-Speech HTTP service with SSML support.
Provides text-to-speech synthesis using Google Cloud's HTTP API with
comprehensive SSML support for voice customization, prosody control,
and styling options. Ideal for applications requiring fine-grained
control over speech output.
Note:
Requires Google Cloud credentials via service account JSON, credentials file,
or default application credentials (GOOGLE_APPLICATION_CREDENTIALS).
Chirp and Journey voices don't support SSML and will use plain text input.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Google HTTP TTS voice customization.
Parameters:
pitch: Voice pitch adjustment (e.g., "+2st", "-50%").
rate: Speaking rate adjustment (e.g., "slow", "fast", "125%").
volume: Volume adjustment (e.g., "loud", "soft", "+6dB").
emphasis: Emphasis level for the text.
language: Language for synthesis. Defaults to English.
gender: Voice gender preference.
google_style: Google-specific voice style.
"""
pitch: Optional[str] = None pitch: Optional[str] = None
rate: Optional[str] = None rate: Optional[str] = None
volume: Optional[str] = None volume: Optional[str] = None
@@ -222,6 +261,16 @@ class GoogleHttpTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initializes the Google HTTP TTS service.
Args:
credentials: JSON string containing Google Cloud service account credentials.
credentials_path: Path to Google Cloud service account JSON file.
voice_id: Google TTS voice identifier (e.g., "en-US-Standard-A").
sample_rate: Audio sample rate in Hz. If None, uses default.
params: Voice customization parameters including pitch, rate, volume, etc.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleHttpTTSService.InputParams() params = params or GoogleHttpTTSService.InputParams()
@@ -245,11 +294,20 @@ class GoogleHttpTTSService(TTSService):
def _create_client( def _create_client(
self, credentials: Optional[str], credentials_path: Optional[str] self, credentials: Optional[str], credentials_path: Optional[str]
) -> texttospeech_v1.TextToSpeechAsyncClient: ) -> texttospeech_v1.TextToSpeechAsyncClient:
"""Create authenticated Google Text-to-Speech client.
Args:
credentials: JSON string with service account credentials.
credentials_path: Path to service account JSON file.
Returns:
Authenticated TextToSpeechAsyncClient instance.
Raises:
ValueError: If no valid credentials are provided.
"""
creds: Optional[service_account.Credentials] = None creds: Optional[service_account.Credentials] = None
# Create a Google Cloud service account for the Cloud Text-to-Speech API
# Using either the provided credentials JSON string or the path to a service account JSON
# file, create a Google Cloud service account and use it to authenticate with the API.
if credentials: if credentials:
# Use provided credentials JSON string # Use provided credentials JSON string
json_account_info = json.loads(credentials) json_account_info = json.loads(credentials)
@@ -271,9 +329,22 @@ class GoogleHttpTTSService(TTSService):
return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Google HTTP TTS service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Google TTS language format.
Args:
language: The language to convert.
Returns:
The Google TTS-specific language code, or None if not supported.
"""
return language_to_google_tts_language(language) return language_to_google_tts_language(language)
def _construct_ssml(self, text: str) -> str: def _construct_ssml(self, text: str) -> str:
@@ -324,6 +395,14 @@ class GoogleHttpTTSService(TTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Google's HTTP TTS API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:
@@ -381,19 +460,13 @@ class GoogleHttpTTSService(TTSService):
class GoogleTTSService(TTSService): class GoogleTTSService(TTSService):
"""Text-to-Speech service using Google Cloud Text-to-Speech API. """Google Cloud Text-to-Speech streaming service.
Converts text to speech using Google's TTS models with streaming synthesis Provides real-time text-to-speech synthesis using Google Cloud's streaming API
for low latency. Supports multiple languages and voices. for low-latency applications. Optimized for Chirp 3 HD and Journey voices
with continuous audio streaming capabilities.
Args: Note:
credentials: JSON string containing Google Cloud service account credentials.
credentials_path: Path to Google Cloud service account JSON file.
voice_id: Google TTS voice identifier (e.g., "en-US-Chirp3-HD-Charon").
sample_rate: Audio sample rate in Hz.
params: Language only.
Notes:
Requires Google Cloud credentials via service account JSON, file path, or Requires Google Cloud credentials via service account JSON, file path, or
default application credentials (GOOGLE_APPLICATION_CREDENTIALS env var). default application credentials (GOOGLE_APPLICATION_CREDENTIALS env var).
Only Chirp 3 HD and Journey voices are supported. Use GoogleHttpTTSService for other voices. Only Chirp 3 HD and Journey voices are supported. Use GoogleHttpTTSService for other voices.
@@ -411,6 +484,12 @@ class GoogleTTSService(TTSService):
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Google streaming TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
def __init__( def __init__(
@@ -423,6 +502,16 @@ class GoogleTTSService(TTSService):
params: InputParams = InputParams(), params: InputParams = InputParams(),
**kwargs, **kwargs,
): ):
"""Initializes the Google streaming TTS service.
Args:
credentials: JSON string containing Google Cloud service account credentials.
credentials_path: Path to Google Cloud service account JSON file.
voice_id: Google TTS voice identifier (e.g., "en-US-Chirp3-HD-Charon").
sample_rate: Audio sample rate in Hz. If None, uses default.
params: Language configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or GoogleTTSService.InputParams() params = params or GoogleTTSService.InputParams()
@@ -466,13 +555,34 @@ class GoogleTTSService(TTSService):
return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Google streaming TTS service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Google TTS language format.
Args:
language: The language to convert.
Returns:
The Google TTS-specific language code, or None if not supported.
"""
return language_to_google_tts_language(language) return language_to_google_tts_language(language)
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate streaming speech from text using Google's streaming API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech as it's generated.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:

View File

@@ -67,12 +67,6 @@ class GrokLLMService(OpenAILLMService):
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Includes specialized token usage tracking that accumulates metrics during Includes specialized token usage tracking that accumulates metrics during
processing and reports final totals. processing and reports final totals.
Args:
api_key: The API key for accessing Grok's API.
base_url: The base URL for Grok API. Defaults to "https://api.x.ai/v1".
model: The model identifier to use. Defaults to "grok-3-beta".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -83,6 +77,14 @@ class GrokLLMService(OpenAILLMService):
model: str = "grok-3-beta", model: str = "grok-3-beta",
**kwargs, **kwargs,
): ):
"""Initialize the GrokLLMService with API key and model.
Args:
api_key: The API key for accessing Grok's API.
base_url: The base URL for Grok API. Defaults to "https://api.x.ai/v1".
model: The model identifier to use. Defaults to "grok-3-beta".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
# Initialize counters for token usage metrics # Initialize counters for token usage metrics
self._prompt_tokens = 0 self._prompt_tokens = 0

View File

@@ -16,12 +16,6 @@ class GroqLLMService(OpenAILLMService):
This service extends OpenAILLMService to connect to Groq's API endpoint while This service extends OpenAILLMService to connect to Groq's API endpoint while
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Args:
api_key: The API key for accessing Groq's API.
base_url: The base URL for Groq API. Defaults to "https://api.groq.com/openai/v1".
model: The model identifier to use. Defaults to "llama-3.3-70b-versatile".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -32,6 +26,14 @@ class GroqLLMService(OpenAILLMService):
model: str = "llama-3.3-70b-versatile", model: str = "llama-3.3-70b-versatile",
**kwargs, **kwargs,
): ):
"""Initialize Groq LLM service.
Args:
api_key: The API key for accessing Groq's API.
base_url: The base URL for Groq API. Defaults to "https://api.groq.com/openai/v1".
model: The model identifier to use. Defaults to "llama-3.3-70b-versatile".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
def create_client(self, api_key=None, base_url=None, **kwargs): def create_client(self, api_key=None, base_url=None, **kwargs):

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Groq speech-to-text service implementation using Whisper models."""
from typing import Optional from typing import Optional
from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription
@@ -15,15 +17,6 @@ class GroqSTTService(BaseWhisperSTTService):
Uses Groq's Whisper API to convert audio to text. Requires a Groq API key Uses Groq's Whisper API to convert audio to text. Requires a Groq API key
set via the api_key parameter or GROQ_API_KEY environment variable. set via the api_key parameter or GROQ_API_KEY environment variable.
Args:
model: Whisper model to use. Defaults to "whisper-large-v3-turbo".
api_key: Groq API key. Defaults to None.
base_url: API base URL. Defaults to "https://api.groq.com/openai/v1".
language: Language of the audio input. Defaults to English.
prompt: Optional text to guide the model's style or continue a previous segment.
temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
**kwargs: Additional arguments passed to BaseWhisperSTTService.
""" """
def __init__( def __init__(
@@ -37,6 +30,17 @@ class GroqSTTService(BaseWhisperSTTService):
temperature: Optional[float] = None, temperature: Optional[float] = None,
**kwargs, **kwargs,
): ):
"""Initialize Groq STT service.
Args:
model: Whisper model to use. Defaults to "whisper-large-v3-turbo".
api_key: Groq API key. Defaults to None.
base_url: API base URL. Defaults to "https://api.groq.com/openai/v1".
language: Language of the audio input. Defaults to English.
prompt: Optional text to guide the model's style or continue a previous segment.
temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
**kwargs: Additional arguments passed to BaseWhisperSTTService.
"""
super().__init__( super().__init__(
model=model, model=model,
api_key=api_key, api_key=api_key,

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Groq text-to-speech service implementation."""
import io import io
import wave import wave
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
@@ -25,7 +27,21 @@ except ModuleNotFoundError as e:
class GroqTTSService(TTSService): class GroqTTSService(TTSService):
"""Groq text-to-speech service implementation.
Provides text-to-speech synthesis using Groq's TTS API. The service
operates at a fixed 48kHz sample rate and supports various voices
and output formats.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Groq TTS configuration.
Parameters:
language: Language for speech synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0 speed: Optional[float] = 1.0
@@ -42,6 +58,17 @@ class GroqTTSService(TTSService):
sample_rate: Optional[int] = GROQ_SAMPLE_RATE, sample_rate: Optional[int] = GROQ_SAMPLE_RATE,
**kwargs, **kwargs,
): ):
"""Initialize Groq TTS service.
Args:
api_key: Groq API key for authentication.
output_format: Audio output format. Defaults to "wav".
params: Additional input parameters for voice customization.
model_name: TTS model to use. Defaults to "playai-tts".
voice_id: Voice identifier to use. Defaults to "Celeste-PlayAI".
sample_rate: Audio sample rate. Must be 48000 Hz for Groq TTS.
**kwargs: Additional arguments passed to parent TTSService class.
"""
if sample_rate != self.GROQ_SAMPLE_RATE: if sample_rate != self.GROQ_SAMPLE_RATE:
logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ") logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ")
@@ -71,10 +98,23 @@ class GroqTTSService(TTSService):
self._client = AsyncGroq(api_key=self._api_key) self._client = AsyncGroq(api_key=self._api_key)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Groq TTS service supports metrics generation.
"""
return True return True
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Groq's TTS API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech data.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
measuring_ttfb = True measuring_ttfb = True
await self.start_ttfb_metrics() await self.start_ttfb_metrics()

View File

@@ -24,12 +24,14 @@ class ImageGenService(AIService):
Processes TextFrames by using their content as prompts for image generation. Processes TextFrames by using their content as prompts for image generation.
Subclasses must implement the run_image_gen method to provide actual image Subclasses must implement the run_image_gen method to provide actual image
generation functionality using their specific AI service. generation functionality using their specific AI service.
Args:
**kwargs: Additional arguments passed to the parent AIService.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
"""Initialize the image generation service.
Args:
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
# Renders the image. Returns an Image object. # Renders the image. Returns an Image object.

View File

@@ -128,11 +128,6 @@ class LLMService(AIService):
parallel and sequential execution modes. Provides event handlers for parallel and sequential execution modes. Provides event handlers for
completion timeouts and function call lifecycle events. completion timeouts and function call lifecycle events.
Args:
run_in_parallel: Whether to run function calls in parallel or sequentially.
Defaults to True.
**kwargs: Additional arguments passed to the parent AIService.
Event handlers: Event handlers:
on_completion_timeout: Called when an LLM completion timeout occurs. on_completion_timeout: Called when an LLM completion timeout occurs.
on_function_calls_started: Called when function calls are received and on_function_calls_started: Called when function calls are received and
@@ -155,6 +150,13 @@ class LLMService(AIService):
adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter
def __init__(self, run_in_parallel: bool = True, **kwargs): def __init__(self, run_in_parallel: bool = True, **kwargs):
"""Initialize the LLM service.
Args:
run_in_parallel: Whether to run function calls in parallel or sequentially.
Defaults to True.
**kwargs: Additional arguments passed to the parent AIService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._run_in_parallel = run_in_parallel self._run_in_parallel = run_in_parallel
self._start_callbacks = {} self._start_callbacks = {}

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""LMNT text-to-speech service implementation."""
import json import json
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
@@ -35,6 +37,14 @@ except ModuleNotFoundError as e:
def language_to_lmnt_language(language: Language) -> Optional[str]: def language_to_lmnt_language(language: Language) -> Optional[str]:
"""Convert a Language enum to LMNT language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding LMNT language code, or None if not supported.
"""
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.DE: "de", Language.DE: "de",
Language.EN: "en", Language.EN: "en",
@@ -71,6 +81,13 @@ def language_to_lmnt_language(language: Language) -> Optional[str]:
class LmntTTSService(InterruptibleTTSService): class LmntTTSService(InterruptibleTTSService):
"""LMNT real-time text-to-speech service.
Provides real-time text-to-speech synthesis using LMNT's WebSocket API.
Supports streaming audio generation with configurable voice models and
language settings.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -81,6 +98,16 @@ class LmntTTSService(InterruptibleTTSService):
model: str = "aurora", model: str = "aurora",
**kwargs, **kwargs,
): ):
"""Initialize the LMNT TTS service.
Args:
api_key: LMNT API key for authentication.
voice_id: ID of the voice to use for synthesis.
sample_rate: Audio sample rate. If None, uses default.
language: Language for synthesis. Defaults to English.
model: TTS model to use. Defaults to "aurora".
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
super().__init__( super().__init__(
push_stop_frames=True, push_stop_frames=True,
pause_frame_processing=True, pause_frame_processing=True,
@@ -99,35 +126,71 @@ class LmntTTSService(InterruptibleTTSService):
self._receive_task = None self._receive_task = None
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as LMNT service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to LMNT service language format.
Args:
language: The language to convert.
Returns:
The LMNT-specific language code, or None if not supported.
"""
return language_to_lmnt_language(language) return language_to_lmnt_language(language)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the LMNT TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the LMNT TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the LMNT TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame downstream with special handling for stop conditions.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
await super().push_frame(frame, direction) await super().push_frame(frame, direction)
if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
self._started = False self._started = False
async def _connect(self): async def _connect(self):
"""Connect to LMNT WebSocket and start receive task."""
await self._connect_websocket() await self._connect_websocket()
if self._websocket and not self._receive_task: if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self): async def _disconnect(self):
"""Disconnect from LMNT WebSocket and clean up tasks."""
if self._receive_task: if self._receive_task:
await self.cancel_task(self._receive_task) await self.cancel_task(self._receive_task)
self._receive_task = None self._receive_task = None
@@ -181,11 +244,13 @@ class LmntTTSService(InterruptibleTTSService):
self._websocket = None self._websocket = None
def _get_websocket(self): def _get_websocket(self):
"""Get the WebSocket connection if available."""
if self._websocket: if self._websocket:
return self._websocket return self._websocket
raise Exception("Websocket not connected") raise Exception("Websocket not connected")
async def flush_audio(self): async def flush_audio(self):
"""Flush any pending audio synthesis."""
if not self._websocket or self._websocket.closed: if not self._websocket or self._websocket.closed:
return return
await self._get_websocket().send(json.dumps({"flush": True})) await self._get_websocket().send(json.dumps({"flush": True}))
@@ -216,7 +281,14 @@ class LmntTTSService(InterruptibleTTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text.""" """Generate TTS audio from text using LMNT's streaming API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:

View File

@@ -35,10 +35,6 @@ class MCPClient(BaseObject):
to LLMs. Supports both stdio and SSE server connections with automatic tool to LLMs. Supports both stdio and SSE server connections with automatic tool
registration and schema conversion. registration and schema conversion.
Args:
server_params: Server connection parameters (stdio or SSE).
**kwargs: Additional arguments passed to the parent BaseObject.
Raises: Raises:
TypeError: If server_params is not a supported parameter type. TypeError: If server_params is not a supported parameter type.
""" """
@@ -48,6 +44,12 @@ class MCPClient(BaseObject):
server_params: Tuple[StdioServerParameters, SseServerParameters, StreamableHttpParameters], server_params: Tuple[StdioServerParameters, SseServerParameters, StreamableHttpParameters],
**kwargs, **kwargs,
): ):
"""Initialize the MCP client with server parameters.
Args:
server_params: Server connection parameters (stdio or SSE).
**kwargs: Additional arguments passed to the parent BaseObject.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self._server_params = server_params self._server_params = server_params
self._session = ClientSession self._session = ClientSession
@@ -190,6 +192,7 @@ class MCPClient(BaseObject):
async def _streamable_http_register_tools(self, llm) -> ToolsSchema: async def _streamable_http_register_tools(self, llm) -> ToolsSchema:
"""Register all available mcp tools with the LLM service using streamable HTTP. """Register all available mcp tools with the LLM service using streamable HTTP.
Args: Args:
llm: The Pipecat LLM service to register tools with llm: The Pipecat LLM service to register tools with
Returns: Returns:

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Mem0 memory service integration for Pipecat.
This module provides a memory service that integrates with Mem0 to store
and retrieve conversational memories, enhancing LLM context with relevant
historical information.
"""
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional
from loguru import logger from loguru import logger
@@ -31,14 +38,21 @@ class Mem0MemoryService(FrameProcessor):
This service intercepts message frames in the pipeline, stores them in Mem0, This service intercepts message frames in the pipeline, stores them in Mem0,
and enhances context with relevant memories before passing them downstream. and enhances context with relevant memories before passing them downstream.
Supports both local and cloud-based Mem0 configurations.
Args:
api_key (str): The API key for accessing Mem0's API
user_id (str): The user ID to associate with memories in Mem0
params (InputParams, optional): Configuration parameters for memory retrieval
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Configuration parameters for Mem0 memory service.
Parameters:
search_limit: Maximum number of memories to retrieve per query.
search_threshold: Minimum similarity threshold for memory retrieval.
api_version: API version to use for Mem0 client operations.
system_prompt: Prefix text for memory context messages.
add_as_system_message: Whether to add memories as system messages.
position: Position to insert memory messages in context.
"""
search_limit: int = Field(default=10, ge=1) search_limit: int = Field(default=10, ge=1)
search_threshold: float = Field(default=0.1, ge=0.0, le=1.0) search_threshold: float = Field(default=0.1, ge=0.0, le=1.0)
api_version: str = Field(default="v2") api_version: str = Field(default="v2")
@@ -56,6 +70,19 @@ class Mem0MemoryService(FrameProcessor):
run_id: Optional[str] = None, run_id: Optional[str] = None,
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
): ):
"""Initialize the Mem0 memory service.
Args:
api_key: The API key for accessing Mem0's cloud API.
local_config: Local configuration for Mem0 client (alternative to cloud API).
user_id: The user ID to associate with memories in Mem0.
agent_id: The agent ID to associate with memories in Mem0.
run_id: The run ID to associate with memories in Mem0.
params: Configuration parameters for memory retrieval and storage.
Raises:
ValueError: If none of user_id, agent_id, or run_id are provided.
"""
# Important: Call the parent class __init__ first # Important: Call the parent class __init__ first
super().__init__() super().__init__()
@@ -86,7 +113,7 @@ class Mem0MemoryService(FrameProcessor):
"""Store messages in Mem0. """Store messages in Mem0.
Args: Args:
messages: List of message dictionaries to store messages: List of message dictionaries to store in memory.
""" """
try: try:
logger.debug(f"Storing {len(messages)} messages in Mem0") logger.debug(f"Storing {len(messages)} messages in Mem0")
@@ -110,10 +137,10 @@ class Mem0MemoryService(FrameProcessor):
"""Retrieve relevant memories from Mem0. """Retrieve relevant memories from Mem0.
Args: Args:
query: The query to search for relevant memories query: The query to search for relevant memories.
Returns: Returns:
List of relevant memory dictionaries List of relevant memory dictionaries matching the query.
""" """
try: try:
logger.debug(f"Retrieving memories for query: {query}") logger.debug(f"Retrieving memories for query: {query}")
@@ -154,8 +181,8 @@ class Mem0MemoryService(FrameProcessor):
"""Enhance the LLM context with relevant memories. """Enhance the LLM context with relevant memories.
Args: Args:
context: The OpenAILLMContext to enhance context: The OpenAILLMContext to enhance with memory information.
query: The query to search for relevant memories query: The query to search for relevant memories.
""" """
# Skip if this is the same query we just processed # Skip if this is the same query we just processed
if self.last_query == query: if self.last_query == query:
@@ -184,8 +211,8 @@ class Mem0MemoryService(FrameProcessor):
"""Process incoming frames, intercept context frames for memory integration. """Process incoming frames, intercept context frames for memory integration.
Args: Args:
frame: The incoming frame to process frame: The incoming frame to process.
direction: The direction of frame flow in the pipeline direction: The direction of frame flow in the pipeline.
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""MiniMax text-to-speech service implementation.
This module provides integration with MiniMax's T2A (Text-to-Audio) API
for streaming text-to-speech synthesis.
"""
import json import json
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
@@ -25,6 +31,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts
def language_to_minimax_language(language: Language) -> Optional[str]: def language_to_minimax_language(language: Language) -> Optional[str]:
"""Convert a Language enum to MiniMax language format.
Args:
language: The Language enum value to convert.
Returns:
The corresponding MiniMax language name, or None if not supported.
"""
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.AR: "Arabic", Language.AR: "Arabic",
Language.CS: "Czech", Language.CS: "Czech",
@@ -71,24 +85,18 @@ def language_to_minimax_language(language: Language) -> Optional[str]:
class MiniMaxHttpTTSService(TTSService): class MiniMaxHttpTTSService(TTSService):
"""Text-to-speech service using MiniMax's T2A (Text-to-Audio) API. """Text-to-speech service using MiniMax's T2A (Text-to-Audio) API.
Provides streaming text-to-speech synthesis using MiniMax's HTTP API
with support for various voice settings, emotions, and audio configurations.
Supports real-time audio streaming with configurable voice parameters.
Platform documentation: Platform documentation:
https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643 https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643
Args:
api_key: MiniMax API key for authentication.
group_id: MiniMax Group ID to identify project.
model: TTS model name (default: "speech-02-turbo"). Options include
"speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo".
voice_id: Voice identifier (default: "Calm_Woman").
aiohttp_session: aiohttp.ClientSession for API communication.
sample_rate: Output audio sample rate in Hz (default: None, set from pipeline).
params: Additional configuration parameters.
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Configuration parameters for MiniMax TTS. """Configuration parameters for MiniMax TTS.
Attributes: Parameters:
language: Language for TTS generation. language: Language for TTS generation.
speed: Speech speed (range: 0.5 to 2.0). speed: Speech speed (range: 0.5 to 2.0).
volume: Speech volume (range: 0 to 10). volume: Speech volume (range: 0 to 10).
@@ -117,6 +125,19 @@ class MiniMaxHttpTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the MiniMax TTS service.
Args:
api_key: MiniMax API key for authentication.
group_id: MiniMax Group ID to identify project.
model: TTS model name. Defaults to "speech-02-turbo". Options include
"speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo".
voice_id: Voice identifier. Defaults to "Calm_Woman".
aiohttp_session: aiohttp.ClientSession for API communication.
sample_rate: Output audio sample rate in Hz. If None, uses pipeline default.
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or MiniMaxHttpTTSService.InputParams() params = params or MiniMaxHttpTTSService.InputParams()
@@ -175,28 +196,62 @@ class MiniMaxHttpTTSService(TTSService):
self._settings["english_normalization"] = params.english_normalization self._settings["english_normalization"] = params.english_normalization
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as MiniMax service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to MiniMax service language format.
Args:
language: The language to convert.
Returns:
The MiniMax-specific language name, or None if not supported.
"""
return language_to_minimax_language(language) return language_to_minimax_language(language)
def set_model_name(self, model: str): def set_model_name(self, model: str):
"""Set the TTS model to use""" """Set the TTS model to use.
Args:
model: The model name to use for synthesis.
"""
self._model_name = model self._model_name = model
def set_voice(self, voice: str): def set_voice(self, voice: str):
"""Set the voice to use""" """Set the voice to use.
Args:
voice: The voice identifier to use for synthesis.
"""
self._voice_id = voice self._voice_id = voice
if "voice_setting" in self._settings: if "voice_setting" in self._settings:
self._settings["voice_setting"]["voice_id"] = voice self._settings["voice_setting"]["voice_id"] = voice
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the MiniMax TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._settings["audio_setting"]["sample_rate"] = self.sample_rate self._settings["audio_setting"]["sample_rate"] = self.sample_rate
logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}") logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}")
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text using MiniMax's streaming API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
headers = { headers = {

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Moondream vision service implementation.
This module provides integration with the Moondream vision-language model
for image analysis and description generation.
"""
import asyncio import asyncio
from typing import AsyncGenerator from typing import AsyncGenerator
@@ -23,7 +29,15 @@ except ModuleNotFoundError as e:
def detect_device(): def detect_device():
"""Detects the appropriate device to run on, and return the device and dtype.""" """Detect the appropriate device to run on.
Detects available hardware acceleration and selects the best device
and data type for optimal performance.
Returns:
tuple: A tuple containing (device, dtype) where device is a torch.device
and dtype is the recommended torch data type for that device.
"""
try: try:
import intel_extension_for_pytorch import intel_extension_for_pytorch
@@ -40,9 +54,24 @@ def detect_device():
class MoondreamService(VisionService): class MoondreamService(VisionService):
"""Moondream vision-language model service.
Provides image analysis and description generation using the Moondream
vision-language model. Supports various hardware acceleration options
including CUDA, MPS, and Intel XPU.
"""
def __init__( def __init__(
self, *, model="vikhyatk/moondream2", revision="2024-08-26", use_cpu=False, **kwargs self, *, model="vikhyatk/moondream2", revision="2024-08-26", use_cpu=False, **kwargs
): ):
"""Initialize the Moondream service.
Args:
model: Hugging Face model identifier for the Moondream model.
revision: Specific model revision to use.
use_cpu: Whether to force CPU usage instead of hardware acceleration.
**kwargs: Additional arguments passed to the parent VisionService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
self.set_model_name(model) self.set_model_name(model)
@@ -65,6 +94,15 @@ class MoondreamService(VisionService):
logger.debug("Loaded Moondream model") logger.debug("Loaded Moondream model")
async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]:
"""Analyze an image and generate a description.
Args:
frame: Vision frame containing the image data and optional question text.
Yields:
Frame: TextFrame containing the generated image description, or ErrorFrame
if analysis fails.
"""
if not self._model: if not self._model:
logger.error(f"{self} error: Moondream model not available ({self.model_name})") logger.error(f"{self} error: Moondream model not available ({self.model_name})")
yield ErrorFrame("Moondream model not available") yield ErrorFrame("Moondream model not available")
@@ -73,6 +111,14 @@ class MoondreamService(VisionService):
logger.debug(f"Analyzing image: {frame}") logger.debug(f"Analyzing image: {frame}")
def get_image_description(frame: VisionImageRawFrame): def get_image_description(frame: VisionImageRawFrame):
"""Generate description for the given image frame.
Args:
frame: Vision frame containing image data and question.
Returns:
str: Generated description of the image.
"""
image = Image.frombytes(frame.format, frame.size, frame.image) image = Image.frombytes(frame.format, frame.size, frame.image)
image_embeds = self._model.encode_image(image) image_embeds = self._model.encode_image(image)
description = self._model.answer_question( description = self._model.answer_question(

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Neuphonic text-to-speech service implementations.
This module provides WebSocket and HTTP-based integrations with Neuphonic's
text-to-speech API for real-time audio synthesis.
"""
import asyncio import asyncio
import base64 import base64
import json import json
@@ -42,6 +48,14 @@ except ModuleNotFoundError as e:
def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
"""Convert a Language enum to Neuphonic language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding Neuphonic language code, or None if not supported.
"""
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.DE: "de", Language.DE: "de",
Language.EN: "en", Language.EN: "en",
@@ -69,7 +83,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]:
class NeuphonicTTSService(InterruptibleTTSService): class NeuphonicTTSService(InterruptibleTTSService):
"""Neuphonic real-time text-to-speech service using WebSocket streaming.
Provides real-time text-to-speech synthesis using Neuphonic's WebSocket API.
Supports interruption handling, keepalive connections, and configurable voice
parameters for high-quality speech generation.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Neuphonic TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0 speed: Optional[float] = 1.0
@@ -84,6 +112,17 @@ class NeuphonicTTSService(InterruptibleTTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Neuphonic TTS service.
Args:
api_key: Neuphonic API key for authentication.
voice_id: ID of the voice to use for synthesis.
url: WebSocket URL for the Neuphonic API.
sample_rate: Audio sample rate in Hz. Defaults to 22050.
encoding: Audio encoding format. Defaults to "pcm_linear".
params: Additional input parameters for TTS configuration.
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
super().__init__( super().__init__(
aggregate_sentences=True, aggregate_sentences=True,
push_text_frames=False, push_text_frames=False,
@@ -114,12 +153,26 @@ class NeuphonicTTSService(InterruptibleTTSService):
self._keepalive_task = None self._keepalive_task = None
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Neuphonic service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Neuphonic service language format.
Args:
language: The language to convert.
Returns:
The Neuphonic-specific language code, or None if not supported.
"""
return language_to_neuphonic_lang_code(language) return language_to_neuphonic_lang_code(language)
async def _update_settings(self, settings: Mapping[str, Any]): async def _update_settings(self, settings: Mapping[str, Any]):
"""Update service settings and reconnect with new configuration."""
if "voice_id" in settings: if "voice_id" in settings:
self.set_voice(settings["voice_id"]) self.set_voice(settings["voice_id"])
@@ -129,28 +182,56 @@ class NeuphonicTTSService(InterruptibleTTSService):
logger.info(f"Switching TTS to settings: [{self._settings}]") logger.info(f"Switching TTS to settings: [{self._settings}]")
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Neuphonic TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the Neuphonic TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the Neuphonic TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def flush_audio(self): async def flush_audio(self):
"""Flush any pending audio synthesis by sending stop command."""
if self._websocket: if self._websocket:
msg = {"text": "<STOP>"} msg = {"text": "<STOP>"}
await self._websocket.send(json.dumps(msg)) await self._websocket.send(json.dumps(msg))
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame downstream with special handling for stop conditions.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
await super().push_frame(frame, direction) await super().push_frame(frame, direction)
if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
self._started = False self._started = False
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with special handling for speech control.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# If we received a TTSSpeakFrame and the LLM response included text (it # If we received a TTSSpeakFrame and the LLM response included text (it
@@ -164,6 +245,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
await self.resume_processing_frames() await self.resume_processing_frames()
async def _connect(self): async def _connect(self):
"""Connect to Neuphonic WebSocket and start background tasks."""
await self._connect_websocket() await self._connect_websocket()
if self._websocket and not self._receive_task: if self._websocket and not self._receive_task:
@@ -173,6 +255,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
self._keepalive_task = self.create_task(self._keepalive_task_handler()) self._keepalive_task = self.create_task(self._keepalive_task_handler())
async def _disconnect(self): async def _disconnect(self):
"""Disconnect from Neuphonic WebSocket and clean up tasks."""
if self._receive_task: if self._receive_task:
await self.cancel_task(self._receive_task) await self.cancel_task(self._receive_task)
self._receive_task = None self._receive_task = None
@@ -184,6 +267,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
await self._disconnect_websocket() await self._disconnect_websocket()
async def _connect_websocket(self): async def _connect_websocket(self):
"""Establish WebSocket connection to Neuphonic API."""
try: try:
if self._websocket and self._websocket.open: if self._websocket and self._websocket.open:
return return
@@ -209,6 +293,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
await self._call_event_handler("on_connection_error", f"{e}") await self._call_event_handler("on_connection_error", f"{e}")
async def _disconnect_websocket(self): async def _disconnect_websocket(self):
"""Close WebSocket connection and clean up state."""
try: try:
await self.stop_all_metrics() await self.stop_all_metrics()
@@ -222,6 +307,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
self._websocket = None self._websocket = None
async def _receive_messages(self): async def _receive_messages(self):
"""Receive and process messages from Neuphonic WebSocket."""
async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager): async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager):
if isinstance(message, str): if isinstance(message, str):
msg = json.loads(message) msg = json.loads(message)
@@ -233,6 +319,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
await self.push_frame(frame) await self.push_frame(frame)
async def _keepalive_task_handler(self): async def _keepalive_task_handler(self):
"""Handle keepalive messages to maintain WebSocket connection."""
KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3 KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3
while True: while True:
self.reset_watchdog() self.reset_watchdog()
@@ -240,6 +327,7 @@ class NeuphonicTTSService(InterruptibleTTSService):
await self._send_text("") await self._send_text("")
async def _send_text(self, text: str): async def _send_text(self, text: str):
"""Send text to Neuphonic WebSocket for synthesis."""
if self._websocket: if self._websocket:
msg = {"text": text} msg = {"text": text}
logger.debug(f"Sending text to websocket: {msg}") logger.debug(f"Sending text to websocket: {msg}")
@@ -247,6 +335,14 @@ class NeuphonicTTSService(InterruptibleTTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Neuphonic's streaming API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"Generating TTS: [{text}]") logger.debug(f"Generating TTS: [{text}]")
try: try:
@@ -274,19 +370,21 @@ class NeuphonicTTSService(InterruptibleTTSService):
class NeuphonicHttpTTSService(TTSService): class NeuphonicHttpTTSService(TTSService):
"""Neuphonic Text-to-Speech service using HTTP streaming. """Neuphonic text-to-speech service using HTTP streaming.
Args: Provides text-to-speech synthesis using Neuphonic's HTTP API with server-sent
api_key: Neuphonic API key events for streaming audio delivery. Suitable for applications that prefer
voice_id: ID of the voice to use HTTP-based communication over WebSocket connections.
url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com")
sample_rate: Sample rate for audio output (default: 22050Hz)
encoding: Audio encoding format (default: "pcm_linear")
params: Additional parameters for TTS generation including language and speed
**kwargs: Additional keyword arguments passed to the parent class
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Neuphonic HTTP TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0 speed: Optional[float] = 1.0
@@ -301,6 +399,17 @@ class NeuphonicHttpTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Neuphonic HTTP TTS service.
Args:
api_key: Neuphonic API key for authentication.
voice_id: ID of the voice to use for synthesis.
url: Base URL for the Neuphonic HTTP API.
sample_rate: Audio sample rate in Hz. Defaults to 22050.
encoding: Audio encoding format. Defaults to "pcm_linear".
params: Additional input parameters for TTS configuration.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or NeuphonicHttpTTSService.InputParams() params = params or NeuphonicHttpTTSService.InputParams()
@@ -316,12 +425,38 @@ class NeuphonicHttpTTSService(TTSService):
self.set_voice(voice_id) self.set_voice(voice_id)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Neuphonic HTTP service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Neuphonic service language format.
Args:
language: The language to convert.
Returns:
The Neuphonic-specific language code, or None if not supported.
"""
return language_to_neuphonic_lang_code(language)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Neuphonic HTTP TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
async def flush_audio(self): async def flush_audio(self):
"""Flush any pending audio synthesis.
Note:
HTTP-based service doesn't require explicit flushing.
"""
pass pass
@traced_tts @traced_tts
@@ -329,9 +464,10 @@ class NeuphonicHttpTTSService(TTSService):
"""Generate speech from text using Neuphonic streaming API. """Generate speech from text using Neuphonic streaming API.
Args: Args:
text: The text to convert to speech text: The text to convert to speech.
Yields: Yields:
Frames containing audio data and status information Frame: Audio frames containing the synthesized speech and status information.
""" """
logger.debug(f"Generating TTS: [{text}]") logger.debug(f"Generating TTS: [{text}]")

View File

@@ -21,12 +21,6 @@ class NimLLMService(OpenAILLMService):
This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining
compatibility with the OpenAI-style interface. It specifically handles the difference compatibility with the OpenAI-style interface. It specifically handles the difference
in token usage reporting between NIM (incremental) and OpenAI (final summary). in token usage reporting between NIM (incremental) and OpenAI (final summary).
Args:
api_key: The API key for accessing NVIDIA's NIM API.
base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1".
model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -37,6 +31,14 @@ class NimLLMService(OpenAILLMService):
model: str = "nvidia/llama-3.1-nemotron-70b-instruct", model: str = "nvidia/llama-3.1-nemotron-70b-instruct",
**kwargs, **kwargs,
): ):
"""Initialize the NimLLMService.
Args:
api_key: The API key for accessing NVIDIA's NIM API.
base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1".
model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
# Counters for accumulating token usage metrics # Counters for accumulating token usage metrics
self._prompt_tokens = 0 self._prompt_tokens = 0

View File

@@ -14,12 +14,14 @@ class OLLamaLLMService(OpenAILLMService):
This service extends OpenAILLMService to work with locally hosted OLLama models, This service extends OpenAILLMService to work with locally hosted OLLama models,
providing a compatible interface for running large language models locally. providing a compatible interface for running large language models locally.
Args:
model: The OLLama model to use. Defaults to "llama2".
base_url: The base URL for the OLLama API endpoint.
Defaults to "http://localhost:11434/v1".
""" """
def __init__(self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1"): def __init__(self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1"):
"""Initialize OLLama LLM service.
Args:
model: The OLLama model to use. Defaults to "llama2".
base_url: The base URL for the OLLama API endpoint.
Defaults to "http://localhost:11434/v1".
"""
super().__init__(model=model, base_url=base_url, api_key="ollama") super().__init__(model=model, base_url=base_url, api_key="ollama")

View File

@@ -48,16 +48,6 @@ class BaseOpenAILLMService(LLMService):
to an OpenAILLMContext object. The context defines what is sent to the LLM for to an OpenAILLMContext object. The context defines what is sent to the LLM for
completion, including user, assistant, and system messages, as well as tool completion, including user, assistant, and system messages, as well as tool
choices and function call configurations. choices and function call configurations.
Args:
model: The OpenAI model name to use (e.g., "gpt-4.1", "gpt-4o").
api_key: OpenAI API key. If None, uses environment variable.
base_url: Custom base URL for OpenAI API. If None, uses default.
organization: OpenAI organization ID.
project: OpenAI project ID.
default_headers: Additional HTTP headers to include in requests.
params: Input parameters for model configuration and behavior.
**kwargs: Additional arguments passed to the parent LLMService.
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
@@ -103,6 +93,18 @@ class BaseOpenAILLMService(LLMService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the BaseOpenAILLMService.
Args:
model: The OpenAI model name to use (e.g., "gpt-4.1", "gpt-4o").
api_key: OpenAI API key. If None, uses environment variable.
base_url: Custom base URL for OpenAI API. If None, uses default.
organization: OpenAI organization ID.
project: OpenAI project ID.
default_headers: Additional HTTP headers to include in requests.
params: Input parameters for model configuration and behavior.
**kwargs: Additional arguments passed to the parent LLMService.
"""
super().__init__(**kwargs) super().__init__(**kwargs)
params = params or BaseOpenAILLMService.InputParams() params = params or BaseOpenAILLMService.InputParams()

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""OpenAI image generation service implementation.
This module provides integration with OpenAI's DALL-E image generation API
for creating images from text prompts.
"""
import io import io
from typing import AsyncGenerator, Literal, Optional from typing import AsyncGenerator, Literal, Optional
@@ -21,6 +27,13 @@ from pipecat.services.image_service import ImageGenService
class OpenAIImageGenService(ImageGenService): class OpenAIImageGenService(ImageGenService):
"""OpenAI DALL-E image generation service.
Provides image generation capabilities using OpenAI's DALL-E models.
Supports various image sizes and can generate images from text prompts
with configurable quality and style parameters.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -30,6 +43,15 @@ class OpenAIImageGenService(ImageGenService):
image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"], image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"],
model: str = "dall-e-3", model: str = "dall-e-3",
): ):
"""Initialize the OpenAI image generation service.
Args:
api_key: OpenAI API key for authentication.
base_url: Custom base URL for OpenAI API. If None, uses default.
aiohttp_session: HTTP session for downloading generated images.
image_size: Target size for generated images.
model: DALL-E model to use for generation. Defaults to "dall-e-3".
"""
super().__init__() super().__init__()
self.set_model_name(model) self.set_model_name(model)
self._image_size = image_size self._image_size = image_size
@@ -37,6 +59,14 @@ class OpenAIImageGenService(ImageGenService):
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
"""Generate an image from a text prompt using OpenAI's DALL-E.
Args:
prompt: Text description of the image to generate.
Yields:
Frame: URLImageRawFrame containing the generated image data.
"""
logger.debug(f"Generating image from prompt: {prompt}") logger.debug(f"Generating image from prompt: {prompt}")
image = await self._client.images.generate( image = await self._client.images.generate(

View File

@@ -61,11 +61,6 @@ class OpenAILLMService(BaseOpenAILLMService):
Provides a complete OpenAI LLM service with context aggregation support. Provides a complete OpenAI LLM service with context aggregation support.
Uses the BaseOpenAILLMService for core functionality and adds OpenAI-specific Uses the BaseOpenAILLMService for core functionality and adds OpenAI-specific
context aggregator creation. context aggregator creation.
Args:
model: The OpenAI model name to use. Defaults to "gpt-4.1".
params: Input parameters for model configuration.
**kwargs: Additional arguments passed to the parent BaseOpenAILLMService.
""" """
def __init__( def __init__(
@@ -75,6 +70,13 @@ class OpenAILLMService(BaseOpenAILLMService):
params: Optional[BaseOpenAILLMService.InputParams] = None, params: Optional[BaseOpenAILLMService.InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize OpenAI LLM service.
Args:
model: The OpenAI model name to use. Defaults to "gpt-4.1".
params: Input parameters for model configuration.
**kwargs: Additional arguments passed to the parent BaseOpenAILLMService.
"""
super().__init__(model=model, params=params, **kwargs) super().__init__(model=model, params=params, **kwargs)
def create_context_aggregator( def create_context_aggregator(

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""OpenAI Speech-to-Text service implementation using OpenAI's transcription API."""
from typing import Optional from typing import Optional
from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription
@@ -15,15 +17,6 @@ class OpenAISTTService(BaseWhisperSTTService):
Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key
set via the api_key parameter or OPENAI_API_KEY environment variable. set via the api_key parameter or OPENAI_API_KEY environment variable.
Args:
model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe".
api_key: OpenAI API key. Defaults to None.
base_url: API base URL. Defaults to None.
language: Language of the audio input. Defaults to English.
prompt: Optional text to guide the model's style or continue a previous segment.
temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
**kwargs: Additional arguments passed to BaseWhisperSTTService.
""" """
def __init__( def __init__(
@@ -37,6 +30,17 @@ class OpenAISTTService(BaseWhisperSTTService):
temperature: Optional[float] = None, temperature: Optional[float] = None,
**kwargs, **kwargs,
): ):
"""Initialize OpenAI STT service.
Args:
model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe".
api_key: OpenAI API key. Defaults to None.
base_url: API base URL. Defaults to None.
language: Language of the audio input. Defaults to English.
prompt: Optional text to guide the model's style or continue a previous segment.
temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0.
**kwargs: Additional arguments passed to BaseWhisperSTTService.
"""
super().__init__( super().__init__(
model=model, model=model,
api_key=api_key, api_key=api_key,

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""OpenAI text-to-speech service implementation.
This module provides integration with OpenAI's text-to-speech API for
generating high-quality synthetic speech from text input.
"""
from typing import AsyncGenerator, Dict, Literal, Optional from typing import AsyncGenerator, Dict, Literal, Optional
from loguru import logger from loguru import logger
@@ -43,16 +49,8 @@ class OpenAITTSService(TTSService):
"""OpenAI Text-to-Speech service that generates audio from text. """OpenAI Text-to-Speech service that generates audio from text.
This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz. This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz.
Supports multiple voice models and configurable parameters for high-quality
Args: speech synthesis with streaming audio output.
api_key: OpenAI API key. Defaults to None.
voice: Voice ID to use. Defaults to "alloy".
model: TTS model to use. Defaults to "gpt-4o-mini-tts".
sample_rate: Output audio sample rate in Hz. Defaults to None.
**kwargs: Additional keyword arguments passed to TTSService.
The service returns PCM-encoded audio at the specified sample rate.
""" """
OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz
@@ -68,6 +66,17 @@ class OpenAITTSService(TTSService):
instructions: Optional[str] = None, instructions: Optional[str] = None,
**kwargs, **kwargs,
): ):
"""Initialize OpenAI TTS service.
Args:
api_key: OpenAI API key for authentication. If None, uses environment variable.
base_url: Custom base URL for OpenAI API. If None, uses default.
voice: Voice ID to use for synthesis. Defaults to "alloy".
model: TTS model to use. Defaults to "gpt-4o-mini-tts".
sample_rate: Output audio sample rate in Hz. If None, uses OpenAI's default 24kHz.
instructions: Optional instructions to guide voice synthesis behavior.
**kwargs: Additional keyword arguments passed to TTSService.
"""
if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE: if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE:
logger.warning( logger.warning(
f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. " f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. "
@@ -81,13 +90,28 @@ class OpenAITTSService(TTSService):
self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) self._client = AsyncOpenAI(api_key=api_key, base_url=base_url)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as OpenAI TTS service supports metrics generation.
"""
return True return True
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the TTS model to use.
Args:
model: The model name to use for text-to-speech synthesis.
"""
logger.info(f"Switching TTS model to: [{model}]") logger.info(f"Switching TTS model to: [{model}]")
self.set_model_name(model) self.set_model_name(model)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the OpenAI TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
if self.sample_rate != self.OPENAI_SAMPLE_RATE: if self.sample_rate != self.OPENAI_SAMPLE_RATE:
logger.warning( logger.warning(
@@ -97,6 +121,14 @@ class OpenAITTSService(TTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using OpenAI's TTS API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech data.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:
await self.start_ttfb_metrics() await self.start_ttfb_metrics()

View File

@@ -26,12 +26,6 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService):
Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, Extends the OpenAI Realtime service to work with Azure OpenAI endpoints,
using Azure's authentication headers and endpoint format. Provides the same using Azure's authentication headers and endpoint format. Provides the same
real-time audio and text communication capabilities as the base OpenAI service. real-time audio and text communication capabilities as the base OpenAI service.
Args:
api_key: The API key for the Azure OpenAI service.
base_url: The full Azure WebSocket endpoint URL including api-version and deployment.
Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment"
**kwargs: Additional arguments passed to parent OpenAIRealtimeBetaLLMService.
""" """
def __init__( def __init__(
@@ -41,6 +35,14 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService):
base_url: str, base_url: str,
**kwargs, **kwargs,
): ):
"""Initialize Azure Realtime Beta LLM service.
Args:
api_key: The API key for the Azure OpenAI service.
base_url: The full Azure WebSocket endpoint URL including api-version and deployment.
Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment"
**kwargs: Additional arguments passed to parent OpenAIRealtimeBetaLLMService.
"""
super().__init__(base_url=base_url, api_key=api_key, **kwargs) super().__init__(base_url=base_url, api_key=api_key, **kwargs)
self.api_key = api_key self.api_key = api_key
self.base_url = base_url self.base_url = base_url

View File

@@ -37,14 +37,16 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext):
Extends the standard OpenAI LLM context to support real-time session properties, Extends the standard OpenAI LLM context to support real-time session properties,
instruction management, and conversion between standard message formats and instruction management, and conversion between standard message formats and
realtime conversation items. realtime conversation items.
Args:
messages: Initial conversation messages. Defaults to None.
tools: Available function tools. Defaults to None.
**kwargs: Additional arguments passed to parent OpenAILLMContext.
""" """
def __init__(self, messages=None, tools=None, **kwargs): def __init__(self, messages=None, tools=None, **kwargs):
"""Initialize the OpenAIRealtimeLLMContext.
Args:
messages: Initial conversation messages. Defaults to None.
tools: Available function tools. Defaults to None.
**kwargs: Additional arguments passed to parent OpenAILLMContext.
"""
super().__init__(messages=messages, tools=tools, **kwargs) super().__init__(messages=messages, tools=tools, **kwargs)
self.__setup_local() self.__setup_local()

View File

@@ -18,13 +18,7 @@ from pydantic import BaseModel, ConfigDict, Field
class InputAudioTranscription(BaseModel): class InputAudioTranscription(BaseModel):
"""Configuration for audio transcription settings. """Configuration for audio transcription settings."""
Parameters:
model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1").
language: Optional language code for transcription.
prompt: Optional transcription hint text.
"""
model: str = "gpt-4o-transcribe" model: str = "gpt-4o-transcribe"
language: Optional[str] language: Optional[str]
@@ -36,6 +30,13 @@ class InputAudioTranscription(BaseModel):
language: Optional[str] = None, language: Optional[str] = None,
prompt: Optional[str] = None, prompt: Optional[str] = None,
): ):
"""Initialize InputAudioTranscription.
Args:
model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1").
language: Optional language code for transcription.
prompt: Optional transcription hint text.
"""
super().__init__(model=model, language=language, prompt=prompt) super().__init__(model=model, language=language, prompt=prompt)
@@ -881,6 +882,8 @@ class TokenDetails(BaseModel):
audio_tokens: Optional[int] = 0 audio_tokens: Optional[int] = 0
class Config: class Config:
"""Pydantic configuration for TokenDetails."""
extra = "allow" extra = "allow"

View File

@@ -96,17 +96,6 @@ class OpenAIRealtimeBetaLLMService(LLMService):
Implements the OpenAI Realtime API Beta with WebSocket communication for low-latency Implements the OpenAI Realtime API Beta with WebSocket communication for low-latency
bidirectional audio and text interactions. Supports function calling, conversation bidirectional audio and text interactions. Supports function calling, conversation
management, and real-time transcription. management, and real-time transcription.
Args:
api_key: OpenAI API key for authentication.
model: OpenAI model name. Defaults to "gpt-4o-realtime-preview-2025-06-03".
base_url: WebSocket base URL for the realtime API.
Defaults to "wss://api.openai.com/v1/realtime".
session_properties: Configuration properties for the realtime session.
If None, uses default SessionProperties.
start_audio_paused: Whether to start with audio input paused. Defaults to False.
send_transcription_frames: Whether to emit transcription frames. Defaults to True.
**kwargs: Additional arguments passed to parent LLMService.
""" """
# Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one. # Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one.
@@ -123,6 +112,19 @@ class OpenAIRealtimeBetaLLMService(LLMService):
send_transcription_frames: bool = True, send_transcription_frames: bool = True,
**kwargs, **kwargs,
): ):
"""Initialize the OpenAI Realtime Beta LLM service.
Args:
api_key: OpenAI API key for authentication.
model: OpenAI model name. Defaults to "gpt-4o-realtime-preview-2025-06-03".
base_url: WebSocket base URL for the realtime API.
Defaults to "wss://api.openai.com/v1/realtime".
session_properties: Configuration properties for the realtime session.
If None, uses default SessionProperties.
start_audio_paused: Whether to start with audio input paused. Defaults to False.
send_transcription_frames: Whether to emit transcription frames. Defaults to True.
**kwargs: Additional arguments passed to parent LLMService.
"""
full_url = f"{base_url}?model={model}" full_url = f"{base_url}?model={model}"
super().__init__(base_url=full_url, **kwargs) super().__init__(base_url=full_url, **kwargs)

View File

@@ -33,15 +33,6 @@ class OpenPipeLLMService(OpenAILLMService):
Extends OpenAI's LLM service to integrate with OpenPipe's fine-tuning and Extends OpenAI's LLM service to integrate with OpenPipe's fine-tuning and
monitoring platform. Provides enhanced request logging and tagging capabilities monitoring platform. Provides enhanced request logging and tagging capabilities
for model training and evaluation. for model training and evaluation.
Args:
model: The model name to use. Defaults to "gpt-4.1".
api_key: OpenAI API key for authentication. If None, reads from environment.
base_url: Custom OpenAI API endpoint URL. Uses default if None.
openpipe_api_key: OpenPipe API key for enhanced features. If None, reads from environment.
openpipe_base_url: OpenPipe API endpoint URL. Defaults to "https://app.openpipe.ai/api/v1".
tags: Optional dictionary of tags to apply to all requests for tracking.
**kwargs: Additional arguments passed to parent OpenAILLMService.
""" """
def __init__( def __init__(
@@ -55,6 +46,17 @@ class OpenPipeLLMService(OpenAILLMService):
tags: Optional[Dict[str, str]] = None, tags: Optional[Dict[str, str]] = None,
**kwargs, **kwargs,
): ):
"""Initialize OpenPipe LLM service.
Args:
model: The model name to use. Defaults to "gpt-4.1".
api_key: OpenAI API key for authentication. If None, reads from environment.
base_url: Custom OpenAI API endpoint URL. Uses default if None.
openpipe_api_key: OpenPipe API key for enhanced features. If None, reads from environment.
openpipe_base_url: OpenPipe API endpoint URL. Defaults to "https://app.openpipe.ai/api/v1".
tags: Optional dictionary of tags to apply to all requests for tracking.
**kwargs: Additional arguments passed to parent OpenAILLMService.
"""
super().__init__( super().__init__(
model=model, model=model,
api_key=api_key, api_key=api_key,

View File

@@ -22,13 +22,6 @@ class OpenRouterLLMService(OpenAILLMService):
This service extends OpenAILLMService to connect to OpenRouter's API endpoint while This service extends OpenAILLMService to connect to OpenRouter's API endpoint while
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Args:
api_key: The API key for accessing OpenRouter's API. If None, will attempt
to read from environment variables.
model: The model identifier to use. Defaults to "openai/gpt-4o-2024-11-20".
base_url: The base URL for OpenRouter API. Defaults to "https://openrouter.ai/api/v1".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -39,6 +32,15 @@ class OpenRouterLLMService(OpenAILLMService):
base_url: str = "https://openrouter.ai/api/v1", base_url: str = "https://openrouter.ai/api/v1",
**kwargs, **kwargs,
): ):
"""Initialize the OpenRouter LLM service.
Args:
api_key: The API key for accessing OpenRouter's API. If None, will attempt
to read from environment variables.
model: The model identifier to use. Defaults to "openai/gpt-4o-2024-11-20".
base_url: The base URL for OpenRouter API. Defaults to "https://openrouter.ai/api/v1".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__( super().__init__(
api_key=api_key, api_key=api_key,
base_url=base_url, base_url=base_url,

View File

@@ -27,12 +27,6 @@ class PerplexityLLMService(OpenAILLMService):
This service extends OpenAILLMService to work with Perplexity's API while maintaining This service extends OpenAILLMService to work with Perplexity's API while maintaining
compatibility with the OpenAI-style interface. It specifically handles the difference compatibility with the OpenAI-style interface. It specifically handles the difference
in token usage reporting between Perplexity (incremental) and OpenAI (final summary). in token usage reporting between Perplexity (incremental) and OpenAI (final summary).
Args:
api_key: The API key for accessing Perplexity's API.
base_url: The base URL for Perplexity's API. Defaults to "https://api.perplexity.ai".
model: The model identifier to use. Defaults to "sonar".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -43,6 +37,14 @@ class PerplexityLLMService(OpenAILLMService):
model: str = "sonar", model: str = "sonar",
**kwargs, **kwargs,
): ):
"""Initialize the Perplexity LLM service.
Args:
api_key: The API key for accessing Perplexity's API.
base_url: The base URL for Perplexity's API. Defaults to "https://api.perplexity.ai".
model: The model identifier to use. Defaults to "sonar".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
# Counters for accumulating token usage metrics # Counters for accumulating token usage metrics
self._prompt_tokens = 0 self._prompt_tokens = 0

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Piper TTS service implementation."""
from typing import AsyncGenerator, Optional from typing import AsyncGenerator, Optional
import aiohttp import aiohttp
@@ -24,12 +26,9 @@ from pipecat.utils.tracing.service_decorators import traced_tts
class PiperTTSService(TTSService): class PiperTTSService(TTSService):
"""Piper TTS service implementation. """Piper TTS service implementation.
Provides integration with Piper's TTS server. Provides integration with Piper's HTTP TTS server for text-to-speech
synthesis. Supports streaming audio generation with configurable sample
Args: rates and automatic WAV header removal.
base_url: API base URL
aiohttp_session: aiohttp ClientSession
sample_rate: Output sample rate
""" """
def __init__( def __init__(
@@ -42,6 +41,14 @@ class PiperTTSService(TTSService):
sample_rate: Optional[int] = None, sample_rate: Optional[int] = None,
**kwargs, **kwargs,
): ):
"""Initialize the Piper TTS service.
Args:
base_url: Base URL for the Piper TTS HTTP server.
aiohttp_session: aiohttp ClientSession for making HTTP requests.
sample_rate: Output sample rate. If None, uses the voice model's native rate.
**kwargs: Additional arguments passed to the parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
if base_url.endswith("/"): if base_url.endswith("/"):
@@ -53,17 +60,22 @@ class PiperTTSService(TTSService):
self._settings = {"base_url": base_url} self._settings = {"base_url": base_url}
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Piper service supports metrics generation.
"""
return True return True
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Piper API. """Generate speech from text using Piper's HTTP API.
Args: Args:
text: The text to convert to speech text: The text to convert to speech.
Yields: Yields:
Frames containing audio data and status information Frame: Audio frames containing the synthesized speech and status frames.
""" """
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
headers = { headers = {

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""PlayHT text-to-speech service implementations.
This module provides integration with PlayHT's text-to-speech API
supporting both WebSocket streaming and HTTP-based synthesis.
"""
import io import io
import json import json
import struct import struct
@@ -42,6 +48,14 @@ except ModuleNotFoundError as e:
def language_to_playht_language(language: Language) -> Optional[str]: def language_to_playht_language(language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding PlayHT language code, or None if not supported.
"""
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.AF: "afrikans", Language.AF: "afrikans",
Language.AM: "amharic", Language.AM: "amharic",
@@ -96,7 +110,22 @@ def language_to_playht_language(language: Language) -> Optional[str]:
class PlayHTTTSService(InterruptibleTTSService): class PlayHTTTSService(InterruptibleTTSService):
"""PlayHT WebSocket-based text-to-speech service.
Provides real-time text-to-speech synthesis using PlayHT's WebSocket API.
Supports streaming audio generation with configurable voice engines and
language settings.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for PlayHT TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
seed: Random seed for voice consistency.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0 speed: Optional[float] = 1.0
seed: Optional[int] = None seed: Optional[int] = None
@@ -113,6 +142,18 @@ class PlayHTTTSService(InterruptibleTTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the PlayHT WebSocket TTS service.
Args:
api_key: PlayHT API key for authentication.
user_id: PlayHT user ID for authentication.
voice_url: URL of the voice to use for synthesis.
voice_engine: Voice engine to use. Defaults to "Play3.0-mini".
sample_rate: Audio sample rate. If None, uses default.
output_format: Audio output format. Defaults to "wav".
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent InterruptibleTTSService.
"""
super().__init__( super().__init__(
pause_frame_processing=True, pause_frame_processing=True,
sample_rate=sample_rate, sample_rate=sample_rate,
@@ -140,30 +181,60 @@ class PlayHTTTSService(InterruptibleTTSService):
self.set_voice(voice_url) self.set_voice(voice_url)
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as PlayHT service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT service language format.
Args:
language: The language to convert.
Returns:
The PlayHT-specific language code, or None if not supported.
"""
return language_to_playht_language(language) return language_to_playht_language(language)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the PlayHT TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the PlayHT TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the PlayHT TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def _connect(self): async def _connect(self):
"""Connect to PlayHT WebSocket and start receive task."""
await self._connect_websocket() await self._connect_websocket()
if self._websocket and not self._receive_task: if self._websocket and not self._receive_task:
self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) self._receive_task = self.create_task(self._receive_task_handler(self._report_error))
async def _disconnect(self): async def _disconnect(self):
"""Disconnect from PlayHT WebSocket and clean up tasks."""
if self._receive_task: if self._receive_task:
await self.cancel_task(self._receive_task) await self.cancel_task(self._receive_task)
self._receive_task = None self._receive_task = None
@@ -171,6 +242,7 @@ class PlayHTTTSService(InterruptibleTTSService):
await self._disconnect_websocket() await self._disconnect_websocket()
async def _connect_websocket(self): async def _connect_websocket(self):
"""Connect to PlayHT websocket."""
try: try:
if self._websocket and self._websocket.open: if self._websocket and self._websocket.open:
return return
@@ -194,6 +266,7 @@ class PlayHTTTSService(InterruptibleTTSService):
await self._call_event_handler("on_connection_error", f"{e}") await self._call_event_handler("on_connection_error", f"{e}")
async def _disconnect_websocket(self): async def _disconnect_websocket(self):
"""Disconnect from PlayHT websocket."""
try: try:
await self.stop_all_metrics() await self.stop_all_metrics()
@@ -207,6 +280,7 @@ class PlayHTTTSService(InterruptibleTTSService):
self._websocket = None self._websocket = None
async def _get_websocket_url(self): async def _get_websocket_url(self):
"""Retrieve WebSocket URL from PlayHT API."""
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.post( async with session.post(
"https://api.play.ht/api/v4/websocket-auth", "https://api.play.ht/api/v4/websocket-auth",
@@ -235,16 +309,19 @@ class PlayHTTTSService(InterruptibleTTSService):
raise Exception(f"Failed to get WebSocket URL: {response.status}") raise Exception(f"Failed to get WebSocket URL: {response.status}")
def _get_websocket(self): def _get_websocket(self):
"""Get the WebSocket connection if available."""
if self._websocket: if self._websocket:
return self._websocket return self._websocket
raise Exception("Websocket not connected") raise Exception("Websocket not connected")
async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection):
"""Handle interruption by stopping metrics and clearing request ID."""
await super()._handle_interruption(frame, direction) await super()._handle_interruption(frame, direction)
await self.stop_all_metrics() await self.stop_all_metrics()
self._request_id = None self._request_id = None
async def _receive_messages(self): async def _receive_messages(self):
"""Receive messages from PlayHT websocket."""
async for message in self._get_websocket(): async for message in self._get_websocket():
if isinstance(message, bytes): if isinstance(message, bytes):
# Skip the WAV header message # Skip the WAV header message
@@ -273,6 +350,14 @@ class PlayHTTTSService(InterruptibleTTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text using PlayHT's WebSocket API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:
@@ -316,7 +401,22 @@ class PlayHTTTSService(InterruptibleTTSService):
class PlayHTHttpTTSService(TTSService): class PlayHTHttpTTSService(TTSService):
"""PlayHT HTTP-based text-to-speech service.
Provides text-to-speech synthesis using PlayHT's HTTP API for simpler,
non-streaming synthesis. Suitable for use cases where streaming is not
required and simpler integration is preferred.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for PlayHT HTTP TTS configuration.
Parameters:
language: Language for synthesis. Defaults to English.
speed: Speech speed multiplier. Defaults to 1.0.
seed: Random seed for voice consistency.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed: Optional[float] = 1.0 speed: Optional[float] = 1.0
seed: Optional[int] = None seed: Optional[int] = None
@@ -333,6 +433,18 @@ class PlayHTHttpTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize the PlayHT HTTP TTS service.
Args:
api_key: PlayHT API key for authentication.
user_id: PlayHT user ID for authentication.
voice_url: URL of the voice to use for synthesis.
voice_engine: Voice engine to use. Defaults to "Play3.0-mini".
protocol: Protocol to use ("http" or "ws"). Defaults to "http".
sample_rate: Audio sample rate. If None, uses default.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or PlayHTHttpTTSService.InputParams() params = params or PlayHTHttpTTSService.InputParams()
@@ -369,10 +481,16 @@ class PlayHTHttpTTSService(TTSService):
self.set_voice(voice_url) self.set_voice(voice_url)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the PlayHT HTTP TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._settings["sample_rate"] = self.sample_rate self._settings["sample_rate"] = self.sample_rate
def _create_options(self) -> TTSOptions: def _create_options(self) -> TTSOptions:
"""Create TTSOptions object from current settings."""
language_str = self._settings["language"] language_str = self._settings["language"]
playht_language = None playht_language = None
if language_str: if language_str:
@@ -392,13 +510,34 @@ class PlayHTHttpTTSService(TTSService):
) )
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as PlayHT HTTP service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to PlayHT service language format.
Args:
language: The language to convert.
Returns:
The PlayHT-specific language code, or None if not supported.
"""
return language_to_playht_language(language) return language_to_playht_language(language)
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate TTS audio from text using PlayHT's HTTP API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:

View File

@@ -16,12 +16,6 @@ class QwenLLMService(OpenAILLMService):
This service extends OpenAILLMService to connect to Qwen's API endpoint while This service extends OpenAILLMService to connect to Qwen's API endpoint while
maintaining full compatibility with OpenAI's interface and functionality. maintaining full compatibility with OpenAI's interface and functionality.
Args:
api_key: The API key for accessing Qwen's API (DashScope API key).
base_url: Base URL for Qwen API. Defaults to "https://dashscope-intl.aliyuncs.com/compatible-mode/v1".
model: The model identifier to use. Defaults to "qwen-plus".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
""" """
def __init__( def __init__(
@@ -32,6 +26,14 @@ class QwenLLMService(OpenAILLMService):
model: str = "qwen-plus", model: str = "qwen-plus",
**kwargs, **kwargs,
): ):
"""Initialize the Qwen LLM service.
Args:
api_key: The API key for accessing Qwen's API (DashScope API key).
base_url: Base URL for Qwen API. Defaults to "https://dashscope-intl.aliyuncs.com/compatible-mode/v1".
model: The model identifier to use. Defaults to "qwen-plus".
**kwargs: Additional keyword arguments passed to OpenAILLMService.
"""
super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs)
logger.info(f"Initialized Qwen LLM service with model: {model}") logger.info(f"Initialized Qwen LLM service with model: {model}")

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Rime text-to-speech service implementations.
This module provides both WebSocket and HTTP-based text-to-speech services
using Rime's API for streaming and batch audio synthesis.
"""
import base64 import base64
import json import json
import uuid import uuid
@@ -47,7 +53,7 @@ def language_to_rime_language(language: Language) -> str:
language: The pipecat Language enum value. language: The pipecat Language enum value.
Returns: Returns:
str: Three-letter language code used by Rime (e.g., 'eng' for English). Three-letter language code used by Rime (e.g., 'eng' for English).
""" """
LANGUAGE_MAP = { LANGUAGE_MAP = {
Language.DE: "ger", Language.DE: "ger",
@@ -67,7 +73,15 @@ class RimeTTSService(AudioContextWordTTSService):
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Configuration parameters for Rime TTS service.""" """Configuration parameters for Rime TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
speed_alpha: Speech speed multiplier. Defaults to 1.0.
reduce_latency: Whether to reduce latency at potential quality cost.
pause_between_brackets: Whether to add pauses between bracketed content.
phonemize_between_brackets: Whether to phonemize bracketed content.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed_alpha: Optional[float] = 1.0 speed_alpha: Optional[float] = 1.0
@@ -96,6 +110,8 @@ class RimeTTSService(AudioContextWordTTSService):
model: Model ID to use for synthesis. model: Model ID to use for synthesis.
sample_rate: Audio sample rate in Hz. sample_rate: Audio sample rate in Hz.
params: Additional configuration parameters. params: Additional configuration parameters.
text_aggregator: Custom text aggregator for processing input text.
**kwargs: Additional arguments passed to parent class.
""" """
# Initialize with parent class settings for proper frame handling # Initialize with parent class settings for proper frame handling
super().__init__( super().__init__(
@@ -135,14 +151,30 @@ class RimeTTSService(AudioContextWordTTSService):
self._cumulative_time = 0 # Accumulates time across messages self._cumulative_time = 0 # Accumulates time across messages
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Rime service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> str | None: def language_to_service_language(self, language: Language) -> str | None:
"""Convert pipecat language to Rime language code.""" """Convert pipecat language to Rime language code.
Args:
language: The language to convert.
Returns:
The Rime-specific language code, or None if not supported.
"""
return language_to_rime_language(language) return language_to_rime_language(language)
async def set_model(self, model: str): async def set_model(self, model: str):
"""Update the TTS model.""" """Update the TTS model.
Args:
model: The model name to use for synthesis.
"""
self._model = model self._model = model
await super().set_model(model) await super().set_model(model)
@@ -159,18 +191,30 @@ class RimeTTSService(AudioContextWordTTSService):
return {"operation": "eos"} return {"operation": "eos"}
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the service and establish websocket connection.""" """Start the service and establish websocket connection.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._settings["samplingRate"] = self.sample_rate self._settings["samplingRate"] = self.sample_rate
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the service and close connection.""" """Stop the service and close connection.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel current operation and clean up.""" """Cancel current operation and clean up.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
@@ -261,6 +305,7 @@ class RimeTTSService(AudioContextWordTTSService):
return word_pairs return word_pairs
async def flush_audio(self): async def flush_audio(self):
"""Flush any pending audio synthesis."""
if not self._context_id or not self._websocket: if not self._context_id or not self._websocket:
return return
@@ -310,7 +355,12 @@ class RimeTTSService(AudioContextWordTTSService):
self._context_id = None self._context_id = None
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push frame and handle end-of-turn conditions.""" """Push frame and handle end-of-turn conditions.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
await super().push_frame(frame, direction) await super().push_frame(frame, direction)
if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)):
if isinstance(frame, TTSStoppedFrame): if isinstance(frame, TTSStoppedFrame):
@@ -318,13 +368,13 @@ class RimeTTSService(AudioContextWordTTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text. """Generate speech from text using Rime's streaming API.
Args: Args:
text: The text to convert to speech. text: The text to convert to speech.
Yields: Yields:
Frames containing audio data and timing information. Frame: Audio frames containing the synthesized speech.
""" """
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:
@@ -354,7 +404,24 @@ class RimeTTSService(AudioContextWordTTSService):
class RimeHttpTTSService(TTSService): class RimeHttpTTSService(TTSService):
"""Rime HTTP-based text-to-speech service.
Provides text-to-speech synthesis using Rime's HTTP API for batch processing.
Suitable for use cases where streaming is not required.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Configuration parameters for Rime HTTP TTS service.
Parameters:
language: Language for synthesis. Defaults to English.
pause_between_brackets: Whether to add pauses between bracketed content.
phonemize_between_brackets: Whether to phonemize bracketed content.
inline_speed_alpha: Inline speed control markup.
speed_alpha: Speech speed multiplier. Defaults to 1.0.
reduce_latency: Whether to reduce latency at potential quality cost.
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
pause_between_brackets: Optional[bool] = False pause_between_brackets: Optional[bool] = False
phonemize_between_brackets: Optional[bool] = False phonemize_between_brackets: Optional[bool] = False
@@ -373,6 +440,17 @@ class RimeHttpTTSService(TTSService):
params: Optional[InputParams] = None, params: Optional[InputParams] = None,
**kwargs, **kwargs,
): ):
"""Initialize Rime HTTP TTS service.
Args:
api_key: Rime API key for authentication.
voice_id: ID of the voice to use.
aiohttp_session: Shared aiohttp session for HTTP requests.
model: Model ID to use for synthesis.
sample_rate: Audio sample rate in Hz.
params: Additional configuration parameters.
**kwargs: Additional arguments passed to parent TTSService.
"""
super().__init__(sample_rate=sample_rate, **kwargs) super().__init__(sample_rate=sample_rate, **kwargs)
params = params or RimeHttpTTSService.InputParams() params = params or RimeHttpTTSService.InputParams()
@@ -396,14 +474,34 @@ class RimeHttpTTSService(TTSService):
self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Rime HTTP service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> str | None: def language_to_service_language(self, language: Language) -> str | None:
"""Convert pipecat language to Rime language code.""" """Convert pipecat language to Rime language code.
Args:
language: The language to convert.
Returns:
The Rime-specific language code, or None if not supported.
"""
return language_to_rime_language(language) return language_to_rime_language(language)
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Rime's HTTP API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
headers = { headers = {

Some files were not shown because too many files have changed in this diff Show More