From 0ecfa827e6ed329590f8ddc9f091e7cc5debcea3 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 28 Jun 2025 13:39:45 -0400 Subject: [PATCH] Improve docstrings for services and processors (#2087) --- CONTRIBUTING.md | 45 ++- docs/api/conf.py | 8 +- pyproject.toml | 5 +- .../processors/aggregators/dtmf_aggregator.py | 26 +- src/pipecat/processors/aggregators/gated.py | 26 +- .../aggregators/gated_openai_llm_context.py | 24 +- .../processors/aggregators/llm_response.py | 318 +++++++++++++-- .../aggregators/openai_llm_context.py | 165 +++++++- .../processors/aggregators/sentence.py | 25 +- .../processors/aggregators/user_response.py | 24 ++ .../aggregators/vision_image_frame.py | 29 +- src/pipecat/processors/async_generator.py | 28 ++ .../audio/audio_buffer_processor.py | 47 ++- src/pipecat/processors/consumer_processor.py | 28 +- .../processors/filters/frame_filter.py | 23 ++ .../processors/filters/function_filter.py | 30 ++ .../processors/filters/identity_filter.py | 19 +- src/pipecat/processors/filters/null_filter.py | 25 +- .../processors/filters/stt_mute_filter.py | 70 +++- .../processors/filters/wake_check_filter.py | 49 ++- .../filters/wake_notifier_filter.py | 24 +- src/pipecat/processors/frame_processor.py | 236 +++++++++++ .../processors/frameworks/langchain.py | 35 ++ src/pipecat/processors/frameworks/rtvi.py | 382 ++++++++++++++++-- .../processors/gstreamer/pipeline_source.py | 41 ++ .../processors/idle_frame_processor.py | 29 +- src/pipecat/processors/logger.py | 23 ++ .../metrics/frame_processor_metrics.py | 72 +++- src/pipecat/processors/metrics/sentry.py | 41 +- src/pipecat/processors/producer_processor.py | 45 ++- src/pipecat/processors/text_transformer.py | 26 +- .../processors/transcript_processor.py | 46 ++- src/pipecat/processors/user_idle_processor.py | 39 +- src/pipecat/services/ai_service.py | 8 +- src/pipecat/services/anthropic/llm.py | 30 +- src/pipecat/services/assemblyai/models.py | 68 +++- src/pipecat/services/assemblyai/stt.py | 57 +++ src/pipecat/services/aws/llm.py | 36 +- src/pipecat/services/aws/stt.py | 70 +++- src/pipecat/services/aws/tts.py | 73 ++++ src/pipecat/services/aws/utils.py | 107 ++++- src/pipecat/services/aws_nova_sonic/aws.py | 42 +- .../services/aws_nova_sonic/context.py | 21 +- src/pipecat/services/azure/common.py | 12 +- src/pipecat/services/azure/image.py | 32 ++ src/pipecat/services/azure/llm.py | 16 +- src/pipecat/services/azure/stt.py | 60 +++ src/pipecat/services/azure/tts.py | 110 ++++- src/pipecat/services/cartesia/stt.py | 99 +++++ src/pipecat/services/cartesia/tts.py | 54 +-- src/pipecat/services/cerebras/llm.py | 14 +- src/pipecat/services/deepgram/stt.py | 20 +- src/pipecat/services/deepgram/tts.py | 36 ++ src/pipecat/services/deepseek/llm.py | 14 +- src/pipecat/services/elevenlabs/tts.py | 192 ++++++++- src/pipecat/services/fal/image.py | 43 ++ src/pipecat/services/fal/stt.py | 56 ++- src/pipecat/services/fireworks/llm.py | 14 +- src/pipecat/services/fish/tts.py | 65 +++ .../services/gemini_multimodal_live/gemini.py | 37 +- src/pipecat/services/gladia/config.py | 20 +- src/pipecat/services/gladia/stt.py | 72 +++- src/pipecat/services/google/frames.py | 40 ++ src/pipecat/services/google/image.py | 41 +- src/pipecat/services/google/llm.py | 32 +- src/pipecat/services/google/llm_openai.py | 27 +- src/pipecat/services/google/llm_vertex.py | 52 ++- src/pipecat/services/google/rtvi.py | 41 ++ src/pipecat/services/google/stt.py | 70 +++- src/pipecat/services/google/tts.py | 140 ++++++- src/pipecat/services/grok/llm.py | 14 +- src/pipecat/services/groq/llm.py | 14 +- src/pipecat/services/groq/stt.py | 22 +- src/pipecat/services/groq/tts.py | 40 ++ src/pipecat/services/image_service.py | 8 +- src/pipecat/services/llm_service.py | 12 +- src/pipecat/services/lmnt/tts.py | 74 +++- src/pipecat/services/mcp_service.py | 11 +- src/pipecat/services/mem0/memory.py | 51 ++- src/pipecat/services/minimax/tts.py | 81 +++- src/pipecat/services/moondream/vision.py | 48 ++- src/pipecat/services/neuphonic/tts.py | 158 +++++++- src/pipecat/services/nim/llm.py | 14 +- src/pipecat/services/ollama/llm.py | 12 +- src/pipecat/services/openai/base_llm.py | 22 +- src/pipecat/services/openai/image.py | 30 ++ src/pipecat/services/openai/llm.py | 12 +- src/pipecat/services/openai/stt.py | 22 +- src/pipecat/services/openai/tts.py | 52 ++- .../services/openai_realtime_beta/azure.py | 14 +- .../services/openai_realtime_beta/context.py | 12 +- .../services/openai_realtime_beta/events.py | 17 +- .../services/openai_realtime_beta/openai.py | 24 +- src/pipecat/services/openpipe/llm.py | 20 +- src/pipecat/services/openrouter/llm.py | 16 +- src/pipecat/services/perplexity/llm.py | 14 +- src/pipecat/services/piper/tts.py | 30 +- src/pipecat/services/playht/tts.py | 139 +++++++ src/pipecat/services/qwen/llm.py | 14 +- src/pipecat/services/rime/tts.py | 120 +++++- src/pipecat/services/riva/stt.py | 156 ++++++- src/pipecat/services/riva/tts.py | 64 +++ src/pipecat/services/sambanova/llm.py | 14 +- src/pipecat/services/sambanova/stt.py | 22 +- src/pipecat/services/sarvam/tts.py | 68 +++- src/pipecat/services/simli/video.py | 26 ++ src/pipecat/services/stt_service.py | 28 +- src/pipecat/services/tavus/video.py | 84 +++- src/pipecat/services/together/llm.py | 14 +- src/pipecat/services/tts_service.py | 84 ++-- src/pipecat/services/ultravox/stt.py | 89 ++-- src/pipecat/services/vision_service.py | 8 +- src/pipecat/services/websocket_service.py | 10 +- src/pipecat/services/whisper/base_stt.py | 62 ++- src/pipecat/services/whisper/stt.py | 103 +++-- src/pipecat/services/xtts/tts.py | 57 +++ src/pipecat/transcriptions/language.py | 18 + 117 files changed, 5136 insertions(+), 862 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 677dc6b7f..1e0594d64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,8 +43,8 @@ We follow Google-style docstrings with these specific conventions: **Regular Classes:** -- Class docstring describes the class purpose and documents all `__init__` parameters in an `Args:` section -- No separate `__init__` docstring needed +- Class docstring describes the class purpose and key functionality +- `__init__` method has its own docstring with complete `Args:` section documenting all parameters - All public methods must have docstrings with `Args:` and `Returns:` sections as appropriate **Dataclasses:** @@ -60,6 +60,17 @@ We follow Google-style docstrings with these specific conventions: - Must have docstrings explaining what subclasses should implement +**`__init__.py` Files:** + +- **Skip docstrings** for pure import/re-export modules +- **Add brief docstrings** for top-level packages or those with initialization logic + +**Enums:** + +- Class docstring describes the enumeration purpose +- Use `Parameters:` section to document each enum value and its meaning +- No `__init__` docstring (Enums don't have custom constructors) + #### Examples: ```python @@ -67,14 +78,18 @@ We follow Google-style docstrings with these specific conventions: class MyService(BaseService): """Description of what the service does. - Args: - param1: Description of param1. - param2: Description of param2. Defaults to True. - **kwargs: Additional arguments passed to parent. + Provides detailed explanation of the service's functionality, + key features, and usage patterns. """ def __init__(self, param1: str, param2: bool = True, **kwargs): - # No docstring - parameters documented above + """Initialize the service. + + Args: + param1: Description of param1. + param2: Description of param2. Defaults to True. + **kwargs: Additional arguments passed to parent. + """ super().__init__(**kwargs) @property @@ -111,6 +126,22 @@ class ConfigParams: host: str port: int = 8080 timeout: float = 30.0 + +# Enum class +class Status(Enum): + """Status codes for processing operations. + + Parameters: + PENDING: Operation is queued but not started. + RUNNING: Operation is currently in progress. + COMPLETED: Operation finished successfully. + FAILED: Operation encountered an error. + """ + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" ``` # Contributor Covenant Code of Conduct diff --git a/docs/api/conf.py b/docs/api/conf.py index a2a568134..b69c62bbb 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -28,15 +28,14 @@ extensions = [ # Napoleon settings napoleon_google_docstring = True -napoleon_numpy_docstring = False -napoleon_include_init_with_doc = False +napoleon_include_init_with_doc = True # AutoDoc settings autodoc_default_options = { "members": True, "member-order": "bysource", "undoc-members": True, - "exclude-members": "__weakref__,__init__", + "exclude-members": "__weakref__,model_config", "no-index": True, "show-inheritance": True, } @@ -173,7 +172,7 @@ autodoc_mock_imports = [ # HTML output settings html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] -autodoc_typehints = "description" +autodoc_typehints = "signature" # Show type hints in the signature only, not in the docstring html_show_sphinx = False @@ -275,6 +274,7 @@ def clean_title(title: str) -> str: "stt": "STT", "tts": "TTS", "llm": "LLM", + "rtvi": "RTVI", } # Check if the entire title is a special case diff --git a/pyproject.toml b/pyproject.toml index 6b1756e7d..f402ddfd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,8 +123,9 @@ select = [ "D", # Docstring rules "I", # Import rules ] -# Ignore requirement for __init__ docstrings -ignore = ["D107"] + +[tool.ruff.lint.per-file-ignores] +"**/__init__.py" = ["D104"] [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/src/pipecat/processors/aggregators/dtmf_aggregator.py b/src/pipecat/processors/aggregators/dtmf_aggregator.py index 3008fb398..006a7c181 100644 --- a/src/pipecat/processors/aggregators/dtmf_aggregator.py +++ b/src/pipecat/processors/aggregators/dtmf_aggregator.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""DTMF aggregation processor for converting keypad input to transcription. + +This module provides a frame processor that aggregates DTMF (Dual-Tone Multi-Frequency) +keypad inputs into meaningful sequences and converts them to transcription frames +for downstream processing by LLM context aggregators. +""" + import asyncio from typing import Optional @@ -31,11 +38,6 @@ class DTMFAggregator(FrameProcessor): - EndFrame or CancelFrame is received Emits TranscriptionFrame for compatibility with existing LLM context aggregators. - - Args: - timeout: Idle timeout in seconds before flushing - termination_digit: Digit that triggers immediate flush - prefix: Prefix added to DTMF sequence in transcription """ def __init__( @@ -45,6 +47,14 @@ class DTMFAggregator(FrameProcessor): prefix: str = "DTMF: ", **kwargs, ): + """Initialize the DTMF aggregator. + + Args: + timeout: Idle timeout in seconds before flushing + termination_digit: Digit that triggers immediate flush + prefix: Prefix added to DTMF sequence in transcription + **kwargs: Additional arguments passed to FrameProcessor + """ super().__init__(**kwargs) self._aggregation = "" self._idle_timeout = timeout @@ -55,6 +65,12 @@ class DTMFAggregator(FrameProcessor): self._aggregation_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection) -> None: + """Process incoming frames and handle DTMF aggregation. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index 3c8aac26e..33f80247d 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Gated frame aggregator for conditional frame accumulation. + +This module provides a gated aggregator that accumulates frames based on +custom gate open/close functions, allowing for conditional frame buffering +and release in frame processing pipelines. +""" + from typing import List, Tuple from loguru import logger @@ -14,8 +21,11 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class GatedAggregator(FrameProcessor): """Accumulate frames, with custom functions to start and stop accumulation. + Yields gate-opening frame before any accumulated frames, then ensuing frames - until and not including the gate-closed frame. + until and not including the gate-closed frame. The aggregator maintains an + internal gate state that controls whether frames are passed through immediately + or accumulated for later release. Doctest: FIXME to work with asyncio >>> from pipecat.frames.frames import ImageRawFrame @@ -48,6 +58,14 @@ class GatedAggregator(FrameProcessor): start_open, direction: FrameDirection = FrameDirection.DOWNSTREAM, ): + """Initialize the gated aggregator. + + Args: + gate_open_fn: Function that returns True when a frame should open the gate. + gate_close_fn: Function that returns True when a frame should close the gate. + start_open: Whether the gate should start in the open state. + direction: The frame direction this aggregator operates on. + """ super().__init__() self._gate_open_fn = gate_open_fn self._gate_close_fn = gate_close_fn @@ -56,6 +74,12 @@ class GatedAggregator(FrameProcessor): self._accumulator: List[Tuple[Frame, FrameDirection]] = [] async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames with gated accumulation logic. + + Args: + frame: The frame to process. + direction: The direction of the frame flow. + """ await super().process_frame(frame, direction) # We must not block system frames. diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index 9973e3d02..56423403d 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Gated OpenAI LLM context aggregator for controlled message flow.""" + from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -11,12 +13,21 @@ from pipecat.sync.base_notifier import BaseNotifier class GatedOpenAILLMContextAggregator(FrameProcessor): - """This aggregator keeps the last received OpenAI LLM context frame and it - doesn't let it through until the notifier is notified. + """Aggregator that gates OpenAI LLM context frames until notified. + This aggregator captures OpenAI LLM context frames and holds them until + a notifier signals that they can be released. This is useful for controlling + the flow of context frames based on external conditions or timing. """ def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs): + """Initialize the gated context aggregator. + + Args: + notifier: The notifier that controls when frames are released. + start_open: If True, the first context frame passes through immediately. + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) self._notifier = notifier self._start_open = start_open @@ -24,6 +35,12 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): self._gate_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames, gating OpenAI LLM context frames. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -42,15 +59,18 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): await self.push_frame(frame, direction) async def _start(self): + """Start the gate task handler.""" if not self._gate_task: self._gate_task = self.create_task(self._gate_task_handler()) async def _stop(self): + """Stop the gate task handler.""" if self._gate_task: await self.cancel_task(self._gate_task) self._gate_task = None async def _gate_task_handler(self): + """Handle the gating logic by waiting for notifications and releasing frames.""" while True: await self._notifier.wait() if self._last_context_frame: diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 40016aaa0..6d27d1ddf 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""LLM response aggregators for handling conversation context and message aggregation. + +This module provides aggregators that process and accumulate LLM responses, user inputs, +and conversation context. These aggregators handle the flow between speech-to-text, +LLM processing, and text-to-speech components in conversational AI pipelines. +""" + import asyncio from abc import abstractmethod from dataclasses import dataclass @@ -54,30 +61,55 @@ from pipecat.utils.time import time_now_iso8601 @dataclass class LLMUserAggregatorParams: + """Parameters for configuring LLM user aggregation behavior. + + Parameters: + aggregation_timeout: Maximum time in seconds to wait for additional + transcription content before pushing aggregated result. This + timeout is used only when the transcription is slow to arrive. + """ + aggregation_timeout: float = 0.5 @dataclass class LLMAssistantAggregatorParams: + """Parameters for configuring LLM assistant aggregation behavior. + + Parameters: + expect_stripped_words: Whether to expect and handle stripped words + in text frames by adding spaces between tokens. + """ + expect_stripped_words: bool = True class LLMFullResponseAggregator(FrameProcessor): - """This is an LLM aggregator that aggregates a full LLM completion. It - aggregates LLM text frames (tokens) received between - `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. Every full - completion is returned via the "on_completion" event handler: + """Aggregates complete LLM responses between start and end frames. - @aggregator.event_handler("on_completion") - async def on_completion( - aggregator: LLMFullResponseAggregator, - completion: str, - completed: bool, - ) + This aggregator collects LLM text frames (tokens) received between + `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame` and provides + the complete response via an event handler. + The aggregator provides an "on_completion" event that fires when a full + completion is available: + + @aggregator.event_handler("on_completion") + async def on_completion( + aggregator: LLMFullResponseAggregator, + completion: str, + completed: bool, + ): + # Handle the completion + pass """ def __init__(self, **kwargs): + """Initialize the LLM full response aggregator. + + Args: + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) self._aggregation = "" @@ -86,6 +118,12 @@ class LLMFullResponseAggregator(FrameProcessor): self._register_event_handler("on_completion") async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and aggregate LLM text content. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): @@ -116,83 +154,123 @@ class LLMFullResponseAggregator(FrameProcessor): class BaseLLMResponseAggregator(FrameProcessor): - """This is the base class for all LLM response aggregators. These - aggregators process incoming frames and aggregate content until they are - ready to push the aggregation. In the case of a user, an aggregation might - be a full transcription received from the STT service. + """Base class for all LLM response aggregators. - The LLM response aggregators also keep a store (e.g. a message list or an - LLM context) of the current conversation, that is, it stores the messages - said by the user or by the bot. + These aggregators process incoming frames and aggregate content until they are + ready to push the aggregation downstream. They maintain conversation state + and handle message flow between different components in the pipeline. + The aggregators keep a store (e.g. message list or LLM context) of the current + conversation, storing messages from both users and the bot. """ def __init__(self, **kwargs): + """Initialize the base LLM response aggregator. + + Args: + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) @property @abstractmethod def messages(self) -> List[dict]: - """Returns the messages from the current conversation.""" + """Get the messages from the current conversation. + + Returns: + List of message dictionaries representing the conversation history. + """ pass @property @abstractmethod def role(self) -> str: - """Returns the role (e.g. user, assistant...) for this aggregator.""" + """Get the role for this aggregator. + + Returns: + The role string (e.g. "user", "assistant") for this aggregator. + """ pass @abstractmethod def add_messages(self, messages): - """Add the given messages to the conversation.""" + """Add the given messages to the conversation. + + Args: + messages: Messages to append to the conversation history. + """ pass @abstractmethod def set_messages(self, messages): - """Reset the conversation with the given messages.""" + """Reset the conversation with the given messages. + + Args: + messages: Messages to replace the current conversation history. + """ pass @abstractmethod def set_tools(self, tools): - """Set LLM tools to be used in the current conversation.""" + """Set LLM tools to be used in the current conversation. + + Args: + tools: List of tool definitions for the LLM to use. + """ pass @abstractmethod def set_tool_choice(self, tool_choice): - """Set the tool choice. This should modify the LLM context.""" + """Set the tool choice for the LLM. + + Args: + tool_choice: Tool choice configuration for the LLM context. + """ pass @abstractmethod async def reset(self): - """Reset the internals of this aggregator. This should not modify the - internal messages. + """Reset the internal state of this aggregator. + + This should clear aggregation state but not modify the conversation messages. """ pass @abstractmethod async def handle_aggregation(self, aggregation: str): - """Adds the given aggregation to the aggregator. The aggregator can use - a simple list of message or a context. It doesn't not push any frames. + """Add the given aggregation to the conversation store. + Args: + aggregation: The aggregated text content to add to the conversation. """ pass @abstractmethod async def push_aggregation(self): - """Pushes the current aggregation. For example, iN the case of context - aggregation this might push a new context frame. + """Push the current aggregation downstream. + The specific frame type pushed depends on the aggregator implementation + (e.g. context frame, messages frame). """ pass class LLMContextResponseAggregator(BaseLLMResponseAggregator): - """This is a base LLM aggregator that uses an LLM context to store the - conversation. It pushes `OpenAILLMContextFrame` as an aggregation frame. + """Base LLM aggregator that uses an OpenAI LLM context for conversation storage. + This aggregator maintains conversation state using an OpenAILLMContext and + pushes OpenAILLMContextFrame objects as aggregation frames. It provides + common functionality for context-based conversation management. """ def __init__(self, *, context: OpenAILLMContext, role: str, **kwargs): + """Initialize the context response aggregator. + + Args: + context: The OpenAI LLM context to use for conversation storage. + role: The role this aggregator represents (e.g. "user", "assistant"). + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._context = context self._role = role @@ -201,46 +279,98 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): @property def messages(self) -> List[dict]: + """Get messages from the LLM context. + + Returns: + List of message dictionaries from the context. + """ return self._context.get_messages() @property def role(self) -> str: + """Get the role for this aggregator. + + Returns: + The role string for this aggregator. + """ return self._role @property def context(self): + """Get the OpenAI LLM context. + + Returns: + The OpenAILLMContext instance used by this aggregator. + """ return self._context def get_context_frame(self) -> OpenAILLMContextFrame: + """Create a context frame with the current context. + + Returns: + OpenAILLMContextFrame containing the current context. + """ return OpenAILLMContextFrame(context=self._context) async def push_context_frame(self, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a context frame in the specified direction. + + Args: + direction: The direction to push the frame (upstream or downstream). + """ frame = self.get_context_frame() await self.push_frame(frame, direction) def add_messages(self, messages): + """Add messages to the context. + + Args: + messages: Messages to add to the conversation context. + """ self._context.add_messages(messages) def set_messages(self, messages): + """Set the context messages. + + Args: + messages: Messages to replace the current context messages. + """ self._context.set_messages(messages) def set_tools(self, tools: List): + """Set tools in the context. + + Args: + tools: List of tool definitions to set in the context. + """ self._context.set_tools(tools) def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict): + """Set tool choice in the context. + + Args: + tool_choice: Tool choice configuration for the context. + """ self._context.set_tool_choice(tool_choice) async def reset(self): + """Reset the aggregation state.""" self._aggregation = "" class LLMUserContextAggregator(LLMContextResponseAggregator): - """This is a user LLM aggregator that uses an LLM context to store the - conversation. It aggregates transcriptions from the STT service and it has - logic to handle multiple scenarios where transcriptions are received between - VAD events (`UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame`) or - even outside or no VAD events at all. + """User LLM aggregator that processes speech-to-text transcriptions. + This aggregator handles the complex logic of aggregating user speech transcriptions + from STT services. It manages multiple scenarios including: + - Transcriptions received between VAD events + - Transcriptions received outside VAD events + - Interim vs final transcriptions + - User interruptions during bot speech + - Emulated VAD for whispered or short utterances + + The aggregator uses timeouts to handle cases where transcriptions arrive + after VAD events or when no VAD is available. """ def __init__( @@ -250,6 +380,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): params: Optional[LLMUserAggregatorParams] = None, **kwargs, ): + """Initialize the user context aggregator. + + Args: + context: The OpenAI LLM context for conversation storage. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments. Supports deprecated 'aggregation_timeout'. + """ super().__init__(context=context, role="user", **kwargs) self._params = params or LLMUserAggregatorParams() if "aggregation_timeout" in kwargs: @@ -275,6 +412,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self._aggregation_task = None async def reset(self): + """Reset the aggregation state and interruption strategies.""" await super().reset() self._was_bot_speaking = False self._seen_interim_results = False @@ -282,9 +420,20 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): [await s.reset() for s in self._interruption_strategies] async def handle_aggregation(self, aggregation: str): + """Add the aggregated user text to the context. + + Args: + aggregation: The aggregated user text to add as a user message. + """ self._context.add_message({"role": self.role, "content": aggregation}) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for user speech aggregation and context management. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -339,7 +488,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): await self.push_frame(frame) async def push_aggregation(self): - """Pushes the current aggregation based on interruption strategies and conditions.""" + """Push the current aggregation based on interruption strategies and conditions.""" if len(self._aggregation) > 0: if self.interruption_strategies and self._bot_speaking: should_interrupt = await self._should_interrupt_based_on_strategies() @@ -373,7 +522,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): # await self.push_frame(OpenAILLMContextFrame(self._context)) async def _should_interrupt_based_on_strategies(self) -> bool: - """Check if interruption should occur based on configured strategies.""" + """Check if interruption should occur based on configured strategies. + + Returns: + True if any interruption strategy indicates interruption should occur. + """ async def should_interrupt(strategy: BaseInterruptionStrategy): await strategy.append_text(self._aggregation) @@ -474,9 +627,10 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self._aggregation_event.clear() async def _maybe_emulate_user_speaking(self): - """Emulate user speaking if we got a transcription but it was not - detected by VAD. Only do that if the bot is not speaking. + """Maybe emulate user speaking based on transcription. + Emulate user speaking if we got a transcription but it was not + detected by VAD. Only do that if the bot is not speaking. """ # Check if we received a transcription but VAD was not able to detect # voice (e.g. when you whisper a short utterance). In that case, we need @@ -497,10 +651,17 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): class LLMAssistantContextAggregator(LLMContextResponseAggregator): - """This is an assistant LLM aggregator that uses an LLM context to store the - conversation. It aggregates text frames received between - `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. + """Assistant LLM aggregator that processes bot responses and function calls. + This aggregator handles the complex logic of processing assistant responses including: + - Text frame aggregation between response start/end markers + - Function call lifecycle management + - Context updates with timestamps + - Tool execution and result handling + - Interruption handling during responses + + The aggregator manages function calls in progress and coordinates between + text generation and tool execution phases of LLM responses. """ def __init__( @@ -510,6 +671,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): params: Optional[LLMAssistantAggregatorParams] = None, **kwargs, ): + """Initialize the assistant context aggregator. + + Args: + context: The OpenAI LLM context for conversation storage. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments. Supports deprecated 'expect_stripped_words'. + """ super().__init__(context=context, role="assistant", **kwargs) self._params = params or LLMAssistantAggregatorParams() @@ -534,26 +702,57 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): """Check if there are any function calls currently in progress. Returns: - bool: True if function calls are in progress, False otherwise + True if function calls are in progress, False otherwise. """ return bool(self._function_calls_in_progress) async def handle_aggregation(self, aggregation: str): + """Add the aggregated assistant text to the context. + + Args: + aggregation: The aggregated assistant text to add as an assistant message. + """ self._context.add_message({"role": "assistant", "content": aggregation}) async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + """Handle a function call that is in progress. + + Args: + frame: The function call in progress frame to handle. + """ pass async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle the result of a completed function call. + + Args: + frame: The function call result frame to handle. + """ pass async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + """Handle cancellation of a function call. + + Args: + frame: The function call cancel frame to handle. + """ pass async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle a user image frame associated with a function call. + + Args: + frame: The user image frame to handle. + """ pass async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for assistant response aggregation and function call management. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): @@ -590,6 +789,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self.push_frame(frame, direction) async def push_aggregation(self): + """Push the current assistant aggregation with timestamp.""" if not self._aggregation: return @@ -719,6 +919,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): class LLMUserResponseAggregator(LLMUserContextAggregator): + """User response aggregator that outputs LLMMessagesFrame instead of context frames. + + This aggregator extends LLMUserContextAggregator but pushes LLMMessagesFrame + objects downstream instead of OpenAILLMContextFrame objects. This is useful + when you need message-based output rather than context-based output. + """ + def __init__( self, messages: Optional[List[dict]] = None, @@ -726,9 +933,17 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): params: Optional[LLMUserAggregatorParams] = None, **kwargs, ): + """Initialize the user response aggregator. + + Args: + messages: Initial messages for the conversation context. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs) async def push_aggregation(self): + """Push the aggregated user input as an LLMMessagesFrame.""" if len(self._aggregation) > 0: await self.handle_aggregation(self._aggregation) @@ -741,6 +956,13 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): + """Assistant response aggregator that outputs LLMMessagesFrame instead of context frames. + + This aggregator extends LLMAssistantContextAggregator but pushes LLMMessagesFrame + objects downstream instead of OpenAILLMContextFrame objects. This is useful + when you need message-based output rather than context-based output. + """ + def __init__( self, messages: Optional[List[dict]] = None, @@ -748,9 +970,17 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): params: Optional[LLMAssistantAggregatorParams] = None, **kwargs, ): + """Initialize the assistant response aggregator. + + Args: + messages: Initial messages for the conversation context. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs) async def push_aggregation(self): + """Push the aggregated assistant response as an LLMMessagesFrame.""" if len(self._aggregation) > 0: await self.handle_aggregation(self._aggregation) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 806741c4c..a8520546a 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI LLM context management for Pipecat. + +This module provides classes for managing OpenAI-specific conversation contexts, +including message handling, tool management, and image/audio processing capabilities. +""" + import base64 import copy import io @@ -29,7 +35,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class CustomEncoder(json.JSONEncoder): + """Custom JSON encoder for handling special data types in logging. + + Provides specialized encoding for io.BytesIO objects to display + readable representations in log output instead of raw binary data. + """ + def default(self, obj): + """Encode special objects for JSON serialization. + + Args: + obj: The object to encode. + + Returns: + Encoded representation of the object. + """ if isinstance(obj, io.BytesIO): # Convert the first 8 bytes to an ASCII hex string return f"{obj.getbuffer()[0:8].hex()}..." @@ -37,25 +57,57 @@ class CustomEncoder(json.JSONEncoder): class OpenAILLMContext: + """Manages conversation context for OpenAI LLM interactions. + + Handles message history, tool definitions, tool choices, and multimedia content + for OpenAI API conversations. Provides methods for message manipulation, + content formatting, and integration with various LLM adapters. + """ + def __init__( self, messages: Optional[List[ChatCompletionMessageParam]] = None, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, ): + """Initialize the OpenAI LLM context. + + Args: + messages: Initial list of conversation messages. + tools: Available tools for the LLM to use. + tool_choice: Tool selection strategy for the LLM. + """ self._messages: List[ChatCompletionMessageParam] = messages if messages else [] self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools self._llm_adapter: Optional[BaseLLMAdapter] = None def get_llm_adapter(self) -> Optional[BaseLLMAdapter]: + """Get the current LLM adapter. + + Returns: + The currently set LLM adapter, or None if not set. + """ return self._llm_adapter def set_llm_adapter(self, llm_adapter: BaseLLMAdapter): + """Set the LLM adapter for context processing. + + Args: + llm_adapter: The LLM adapter to use for tool conversion. + """ self._llm_adapter = llm_adapter @staticmethod def from_messages(messages: List[dict]) -> "OpenAILLMContext": + """Create a context from a list of message dictionaries. + + Args: + messages: List of message dictionaries to convert to context. + + Returns: + New OpenAILLMContext instance with the provided messages. + """ context = OpenAILLMContext() for message in messages: @@ -66,34 +118,81 @@ class OpenAILLMContext: @property def messages(self) -> List[ChatCompletionMessageParam]: + """Get the current messages list. + + Returns: + List of conversation messages. + """ return self._messages @property def tools(self) -> List[ChatCompletionToolParam] | NotGiven | List[Any]: + """Get the tools list, converting through adapter if available. + + Returns: + Tools list, potentially converted by the LLM adapter. + """ if self._llm_adapter: return self._llm_adapter.from_standard_tools(self._tools) return self._tools @property def tool_choice(self) -> ChatCompletionToolChoiceOptionParam | NotGiven: + """Get the current tool choice setting. + + Returns: + The tool choice configuration. + """ return self._tool_choice def add_message(self, message: ChatCompletionMessageParam): + """Add a single message to the context. + + Args: + message: The message to add to the conversation history. + """ self._messages.append(message) def add_messages(self, messages: List[ChatCompletionMessageParam]): + """Add multiple messages to the context. + + Args: + messages: List of messages to add to the conversation history. + """ self._messages.extend(messages) def set_messages(self, messages: List[ChatCompletionMessageParam]): + """Replace all messages in the context. + + Args: + messages: New list of messages to replace the current history. + """ self._messages[:] = messages def get_messages(self) -> List[ChatCompletionMessageParam]: + """Get a copy of the current messages list. + + Returns: + List of all messages in the conversation history. + """ return self._messages def get_messages_json(self) -> str: + """Get messages as a formatted JSON string. + + Returns: + JSON string representation of all messages with custom encoding. + """ return json.dumps(self._messages, cls=CustomEncoder, ensure_ascii=False, indent=2) def get_messages_for_logging(self) -> str: + """Get sanitized messages suitable for logging. + + Removes or truncates sensitive data like image content for safe logging. + + Returns: + JSON string with sanitized message content for logging. + """ msgs = [] for message in self.messages: msg = copy.deepcopy(message) @@ -118,10 +217,10 @@ class OpenAILLMContext: Since OpenAI is our standard format, this is a passthrough function. Args: - message (dict): Message in OpenAI format + message: Message in OpenAI format. Returns: - dict: Same message, unchanged + Same message, unchanged. """ return message @@ -133,20 +232,28 @@ class OpenAILLMContext: other LLM services that may need to return multiple messages. Args: - obj (dict): Message in OpenAI format with either: - - Simple content: {"role": "user", "content": "Hello"} - - List content: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + obj: Message in OpenAI format with either simple string content + or structured list content. Returns: - list: List containing the original messages, preserving whether - the content was in simple string or structured list format + List containing the original messages, preserving the content format. """ return [obj] def get_messages_for_initializing_history(self): + """Get messages for initializing conversation history. + + Returns: + List of messages suitable for history initialization. + """ return self._messages def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Returns: + List of messages converted to standard format for storage. + """ messages = [] for m in self._messages: standard_messages = self.to_standard_messages(m) @@ -154,9 +261,19 @@ class OpenAILLMContext: return messages def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven): + """Set the tool choice configuration. + + Args: + tool_choice: Tool selection strategy for the LLM. + """ self._tool_choice = tool_choice def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN): + """Set the available tools for the LLM. + + Args: + tools: List of tools available to the LLM, or NOT_GIVEN to disable tools. + """ if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0: tools = NOT_GIVEN self._tools = tools @@ -164,6 +281,14 @@ class OpenAILLMContext: def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: str = None ): + """Add a message containing an image frame. + + Args: + format: Image format (e.g., 'RGB', 'RGBA'). + size: Image dimensions as (width, height) tuple. + image: Raw image bytes. + text: Optional text to include with the image. + """ buffer = io.BytesIO() Image.frombytes(format, size, image).save(buffer, format="JPEG") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") @@ -177,10 +302,30 @@ class OpenAILLMContext: self.add_message({"role": "user", "content": content}) def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None): + """Add a message containing audio frames. + + Args: + audio_frames: List of audio frame objects to include. + text: Optional text to include with the audio. + + Note: + This method is currently a placeholder for future implementation. + """ # todo: implement for OpenAI models and others pass def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size): + """Create a WAV file header for audio data. + + Args: + sample_rate: Audio sample rate in Hz. + num_channels: Number of audio channels. + bits_per_sample: Bits per audio sample. + data_size: Size of audio data in bytes. + + Returns: + WAV header as a bytearray. + """ # RIFF chunk descriptor header = bytearray() header.extend(b"RIFF") # ChunkID @@ -206,10 +351,14 @@ class OpenAILLMContext: @dataclass class OpenAILLMContextFrame(Frame): - """Like an LLMMessagesFrame, but with extra context specific to the OpenAI + """Frame containing OpenAI-specific LLM context. + + Like an LLMMessagesFrame, but with extra context specific to the OpenAI API. The context in this message is also mutable, and will be changed by the OpenAIContextAggregator frame processor. + Parameters: + context: The OpenAI LLM context containing messages, tools, and configuration. """ context: OpenAILLMContext diff --git a/src/pipecat/processors/aggregators/sentence.py b/src/pipecat/processors/aggregators/sentence.py index 54aeea16a..5a2bc59dc 100644 --- a/src/pipecat/processors/aggregators/sentence.py +++ b/src/pipecat/processors/aggregators/sentence.py @@ -4,17 +4,28 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Text sentence aggregation processor for Pipecat. + +This module provides a frame processor that accumulates text frames into +complete sentences, only outputting when a sentence-ending pattern is detected. +""" + from pipecat.frames.frames import EndFrame, Frame, InterimTranscriptionFrame, TextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.utils.string import match_endofsentence class SentenceAggregator(FrameProcessor): - """This frame processor aggregates text frames into complete sentences. + """Aggregates text frames into complete sentences. + + This processor accumulates incoming text frames until a sentence-ending + pattern is detected, then outputs the complete sentence as a single frame. + Useful for ensuring downstream processors receive coherent, complete sentences + rather than fragmented text. Frame input/output: TextFrame("Hello,") -> None - TextFrame(" world.") -> TextFrame("Hello world.") + TextFrame(" world.") -> TextFrame("Hello, world.") Doctest: FIXME to work with asyncio >>> import asyncio @@ -29,10 +40,20 @@ class SentenceAggregator(FrameProcessor): """ def __init__(self): + """Initialize the sentence aggregator. + + Sets up internal state for accumulating text frames into complete sentences. + """ super().__init__() self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and aggregate text into complete sentences. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) # We ignore interim description at this point. diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index 8831f7d10..87c9d9f58 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -4,15 +4,39 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""User response aggregation for text frames. + +This module provides an aggregator that collects user responses and outputs +them as TextFrame objects, useful for capturing and processing user input +in conversational pipelines. +""" + from pipecat.frames.frames import TextFrame from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator class UserResponseAggregator(LLMUserResponseAggregator): + """Aggregates user responses into TextFrame objects. + + This aggregator extends LLMUserResponseAggregator to specifically handle + user input by collecting text responses and outputting them as TextFrame + objects when the aggregation is complete. + """ + def __init__(self, **kwargs): + """Initialize the user response aggregator. + + Args: + **kwargs: Additional arguments passed to parent LLMUserResponseAggregator. + """ super().__init__(**kwargs) async def push_aggregation(self): + """Push the aggregated user response as a TextFrame. + + Creates a TextFrame from the current aggregation if it contains content, + resets the aggregation state, and pushes the frame downstream. + """ if len(self._aggregation) > 0: frame = TextFrame(self._aggregation.strip()) diff --git a/src/pipecat/processors/aggregators/vision_image_frame.py b/src/pipecat/processors/aggregators/vision_image_frame.py index e5e0e41da..ea1848ff1 100644 --- a/src/pipecat/processors/aggregators/vision_image_frame.py +++ b/src/pipecat/processors/aggregators/vision_image_frame.py @@ -4,14 +4,22 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Vision image frame aggregation for Pipecat. + +This module provides frame aggregation functionality to combine text and image +frames into vision frames for multimodal processing. +""" + from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class VisionImageFrameAggregator(FrameProcessor): - """This aggregator waits for a consecutive TextFrame and an - InputImageRawFrame. After the InputImageRawFrame arrives it will output a - VisionImageRawFrame. + """Aggregates consecutive text and image frames into vision frames. + + This aggregator waits for a consecutive TextFrame and an InputImageRawFrame. + After the InputImageRawFrame arrives it will output a VisionImageRawFrame + combining both the text and image data for multimodal processing. >>> from pipecat.frames.frames import ImageFrame @@ -23,14 +31,27 @@ class VisionImageFrameAggregator(FrameProcessor): >>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?"))) >>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0)))) VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B - """ def __init__(self): + """Initialize the vision image frame aggregator. + + The aggregator starts with no cached text, waiting for the first + TextFrame to arrive before it can create vision frames. + """ super().__init__() self._describe_text = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and aggregate text with images. + + Caches TextFrames and combines them with subsequent InputImageRawFrames + to create VisionImageRawFrames. Other frames are passed through unchanged. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, TextFrame): diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py index 142d9cb47..8e03a989a 100644 --- a/src/pipecat/processors/async_generator.py +++ b/src/pipecat/processors/async_generator.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Async generator processor for frame serialization and streaming.""" + import asyncio from typing import Any, AsyncGenerator @@ -17,12 +19,32 @@ from pipecat.serializers.base_serializer import FrameSerializer class AsyncGeneratorProcessor(FrameProcessor): + """A frame processor that serializes frames and provides them via async generator. + + This processor passes frames through unchanged while simultaneously serializing + them and making the serialized data available through an async generator interface. + Useful for streaming frame data to external consumers while maintaining the + normal frame processing pipeline. + """ + def __init__(self, *, serializer: FrameSerializer, **kwargs): + """Initialize the async generator processor. + + Args: + serializer: The frame serializer to use for converting frames to data. + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) self._serializer = serializer self._data_queue = asyncio.Queue() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames by passing them through and queuing serialized data. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) await self.push_frame(frame, direction) @@ -35,6 +57,12 @@ class AsyncGeneratorProcessor(FrameProcessor): await self._data_queue.put(data) async def generator(self) -> AsyncGenerator[Any, None]: + """Generate serialized frame data asynchronously. + + Yields: + Serialized frame data from the internal queue until a termination + signal (None) is received. + """ running = True while running: data = await self._data_queue.get() diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 13d5a84bc..f48d2d6a8 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Audio buffer processor for managing and synchronizing audio streams. + +This module provides an AudioBufferProcessor that handles buffering and synchronization +of audio from both user input and bot output sources, with support for various audio +configurations and event-driven processing. +""" + import time from typing import Optional @@ -37,12 +44,6 @@ class AudioBufferProcessor(FrameProcessor): on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio - Args: - sample_rate (Optional[int]): Desired output sample rate. If None, uses source rate - num_channels (int): Number of channels (1 for mono, 2 for stereo). Defaults to 1 - buffer_size (int): Size of buffer before triggering events. 0 for no buffering - enable_turn_audio (bool): Whether turn audio event handlers should be triggered - Audio handling: - Mono output (num_channels=1): User and bot audio are mixed - Stereo output (num_channels=2): User audio on left, bot audio on right @@ -61,6 +62,16 @@ class AudioBufferProcessor(FrameProcessor): enable_turn_audio: bool = False, **kwargs, ): + """Initialize the audio buffer processor. + + Args: + sample_rate: Desired output sample rate. If None, uses source rate. + num_channels: Number of channels (1 for mono, 2 for stereo). Defaults to 1. + buffer_size: Size of buffer before triggering events. 0 for no buffering. + user_continuous_stream: Deprecated parameter for backwards compatibility. + enable_turn_audio: Whether turn audio event handlers should be triggered. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._init_sample_rate = sample_rate self._sample_rate = 0 @@ -105,7 +116,7 @@ class AudioBufferProcessor(FrameProcessor): """Current sample rate of the audio processor. Returns: - int: The sample rate in Hz + The sample rate in Hz. """ return self._sample_rate @@ -114,7 +125,7 @@ class AudioBufferProcessor(FrameProcessor): """Number of channels in the audio output. Returns: - int: Number of channels (1 for mono, 2 for stereo) + Number of channels (1 for mono, 2 for stereo). """ return self._num_channels @@ -122,7 +133,7 @@ class AudioBufferProcessor(FrameProcessor): """Check if both user and bot audio buffers contain data. Returns: - bool: True if both buffers contain audio data + True if both buffers contain audio data. """ return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio( self._bot_audio_buffer @@ -135,7 +146,7 @@ class AudioBufferProcessor(FrameProcessor): on the left channel and bot audio on the right channel. Returns: - bytes: Mixed audio data + Mixed audio data as bytes. """ if self._num_channels == 1: return mix_audio(bytes(self._user_audio_buffer), bytes(self._bot_audio_buffer)) @@ -163,7 +174,12 @@ class AudioBufferProcessor(FrameProcessor): self._recording = False async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming audio frames and manage audio buffers.""" + """Process incoming audio frames and manage audio buffers. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) # Update output sample rate if necessary. @@ -181,10 +197,12 @@ class AudioBufferProcessor(FrameProcessor): await self.push_frame(frame, direction) def _update_sample_rate(self, frame: StartFrame): + """Update the sample rate from the start frame.""" self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate self._audio_buffer_size_1s = self._sample_rate * 2 async def _process_recording(self, frame: Frame): + """Process audio frames for recording.""" if isinstance(frame, InputAudioRawFrame): # Add silence if we need to. silence = self._compute_silence(self._last_user_frame_at) @@ -208,6 +226,7 @@ class AudioBufferProcessor(FrameProcessor): await self._call_on_audio_data_handler() async def _process_turn_recording(self, frame: Frame): + """Process frames for turn-based audio recording.""" if isinstance(frame, UserStartedSpeakingFrame): self._user_speaking = True elif isinstance(frame, UserStoppedSpeakingFrame): @@ -242,6 +261,7 @@ class AudioBufferProcessor(FrameProcessor): self._bot_turn_audio_buffer += resampled async def _call_on_audio_data_handler(self): + """Call the audio data event handlers with buffered audio.""" if not self.has_audio() or not self._recording: return @@ -263,23 +283,28 @@ class AudioBufferProcessor(FrameProcessor): self._reset_audio_buffers() def _buffer_has_audio(self, buffer: bytearray) -> bool: + """Check if a buffer contains audio data.""" return buffer is not None and len(buffer) > 0 def _reset_recording(self): + """Reset recording state and buffers.""" self._reset_audio_buffers() self._last_user_frame_at = time.time() self._last_bot_frame_at = time.time() def _reset_audio_buffers(self): + """Reset all audio buffers to empty state.""" self._user_audio_buffer = bytearray() self._bot_audio_buffer = bytearray() self._user_turn_audio_buffer = bytearray() self._bot_turn_audio_buffer = bytearray() async def _resample_audio(self, frame: AudioRawFrame) -> bytes: + """Resample audio frame to the target sample rate.""" return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate) def _compute_silence(self, from_time: float) -> bytes: + """Compute silence to insert based on time gap.""" quiet_time = time.time() - from_time # We should get audio frames very frequently. We introduce silence only # if there's a big enough gap of 1s. diff --git a/src/pipecat/processors/consumer_processor.py b/src/pipecat/processors/consumer_processor.py index 977450181..277cef2cd 100644 --- a/src/pipecat/processors/consumer_processor.py +++ b/src/pipecat/processors/consumer_processor.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Consumer processor for consuming frames from ProducerProcessor queues.""" + import asyncio from typing import Awaitable, Callable, Optional @@ -14,11 +16,11 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue class ConsumerProcessor(FrameProcessor): - """This class passes-through frames and also consumes frames from a - producer's queue. When a frame from a producer queue is received it will be - pushed to the specified direction. The frames can be transformed into a - different type of frame before being pushed. + """Frame processor that consumes frames from a ProducerProcessor's queue. + This processor passes through frames normally while also consuming frames + from a ProducerProcessor's queue. When frames are received from the producer + queue, they are optionally transformed and pushed in the specified direction. """ def __init__( @@ -29,6 +31,14 @@ class ConsumerProcessor(FrameProcessor): direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs, ): + """Initialize the consumer processor. + + Args: + producer: The producer processor to consume frames from. + transformer: Function to transform frames before pushing. Defaults to identity_transformer. + direction: Direction to push consumed frames. Defaults to DOWNSTREAM. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._transformer = transformer self._direction = direction @@ -36,6 +46,12 @@ class ConsumerProcessor(FrameProcessor): self._consumer_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle lifecycle events. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -48,19 +64,23 @@ class ConsumerProcessor(FrameProcessor): await self.push_frame(frame, direction) async def _start(self, _: StartFrame): + """Start the consumer task and register with the producer.""" if not self._consumer_task: self._queue: WatchdogQueue = self._producer.add_consumer() self._consumer_task = self.create_task(self._consumer_task_handler()) async def _stop(self, _: EndFrame): + """Stop the consumer task.""" if self._consumer_task: await self.cancel_task(self._consumer_task) async def _cancel(self, _: CancelFrame): + """Cancel the consumer task.""" if self._consumer_task: await self.cancel_task(self._consumer_task) async def _consumer_task_handler(self): + """Handle consuming frames from the producer queue.""" while True: frame = await self._queue.get() new_frame = await self._transformer(frame) diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py index 43e2dab95..b2084e4df 100644 --- a/src/pipecat/processors/filters/frame_filter.py +++ b/src/pipecat/processors/filters/frame_filter.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame filtering processor for the Pipecat framework.""" + from typing import Tuple, Type from pipecat.frames.frames import EndFrame, Frame, SystemFrame @@ -11,7 +13,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class FrameFilter(FrameProcessor): + """A frame processor that filters frames based on their types. + + This processor acts as a selective gate in the pipeline, allowing only + frames of specified types to pass through. System and end frames are + automatically allowed to pass through to maintain pipeline integrity. + """ + def __init__(self, types: Tuple[Type[Frame], ...]): + """Initialize the frame filter. + + Args: + types: Tuple of frame types that should be allowed to pass through + the filter. All other frame types (except SystemFrame and + EndFrame) will be blocked. + """ super().__init__() self._types = types @@ -20,12 +36,19 @@ class FrameFilter(FrameProcessor): # def _should_passthrough_frame(self, frame): + """Determine if a frame should pass through the filter.""" if isinstance(frame, self._types): return True return isinstance(frame, (EndFrame, SystemFrame)) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process an incoming frame and conditionally pass it through. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if self._should_passthrough_frame(frame): diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index a8fa5b2af..e663b81f4 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Function-based frame filtering for Pipecat pipelines. + +This module provides a processor that filters frames based on a custom function, +allowing for flexible frame filtering logic in processing pipelines. +""" + from typing import Awaitable, Callable from pipecat.frames.frames import EndFrame, Frame, SystemFrame @@ -11,11 +17,26 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class FunctionFilter(FrameProcessor): + """A frame processor that filters frames using a custom function. + + This processor allows frames to pass through based on the result of a + user-provided filter function. System and end frames always pass through + regardless of the filter result. + """ + def __init__( self, filter: Callable[[Frame], Awaitable[bool]], direction: FrameDirection = FrameDirection.DOWNSTREAM, ): + """Initialize the function filter. + + Args: + filter: An async function that takes a Frame and returns True if the + frame should pass through, False otherwise. + direction: The direction to apply filtering. Only frames moving in + this direction will be filtered. Defaults to DOWNSTREAM. + """ super().__init__() self._filter = filter self._direction = direction @@ -27,9 +48,18 @@ class FunctionFilter(FrameProcessor): # Ignore system frames, end frames and frames that are not following the # direction of this gate def _should_passthrough_frame(self, frame, direction): + """Check if a frame should pass through without filtering.""" + # Ignore system frames, end frames and frames that are not following the + # direction of this gate return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process a frame through the filter. + + Args: + frame: The frame to process. + direction: The direction the frame is moving in the pipeline. + """ await super().process_frame(frame, direction) passthrough = self._should_passthrough_frame(frame, direction) diff --git a/src/pipecat/processors/filters/identity_filter.py b/src/pipecat/processors/filters/identity_filter.py index e0bfebcf5..f3999b59c 100644 --- a/src/pipecat/processors/filters/identity_filter.py +++ b/src/pipecat/processors/filters/identity_filter.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Identity filter for transparent frame passthrough. + +This module provides a simple passthrough filter that forwards all frames +without modification, useful for testing and pipeline composition. +""" + from pipecat.frames.frames import Frame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -14,10 +20,14 @@ class IdentityFilter(FrameProcessor): This filter acts as a transparent passthrough, allowing all frames to flow through unchanged. It can be useful when testing `ParallelPipeline` to create pipelines that pass through frames (no frames should be repeated). - """ def __init__(self, **kwargs): + """Initialize the identity filter. + + Args: + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) # @@ -25,6 +35,11 @@ class IdentityFilter(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process an incoming frame by passing it through unchanged.""" + """Process an incoming frame by passing it through unchanged. + + Args: + frame: The frame to process and forward. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/filters/null_filter.py b/src/pipecat/processors/filters/null_filter.py index 8e6a6dda8..1b6670aef 100644 --- a/src/pipecat/processors/filters/null_filter.py +++ b/src/pipecat/processors/filters/null_filter.py @@ -4,14 +4,31 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Null filter processor for blocking frame transmission. + +This module provides a frame processor that blocks all frames except +system and end frames, useful for testing or temporarily stopping +frame flow in a pipeline. +""" + from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class NullFilter(FrameProcessor): - """This filter doesn't allow passing any frames up or downstream.""" + """A filter that blocks all frames except system and end frames. + + This processor acts as a null filter, preventing frames from passing + through the pipeline while still allowing essential system and end + frames to maintain proper pipeline operation. + """ def __init__(self, **kwargs): + """Initialize the null filter. + + Args: + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) # @@ -19,6 +36,12 @@ class NullFilter(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames, only allowing system and end frames through. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, (SystemFrame, EndFrame)): diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index e85a3e581..700313d31 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -39,12 +39,17 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class STTMuteStrategy(Enum): """Strategies determining when STT should be muted. - Attributes: - FIRST_SPEECH: Mute only during first detected bot speech - MUTE_UNTIL_FIRST_BOT_COMPLETE: Start muted and remain muted until first bot speech completes - FUNCTION_CALL: Mute during function calls - ALWAYS: Mute during all bot speech - CUSTOM: Allow custom logic via callback + Each strategy defines different conditions under which speech-to-text + processing should be temporarily disabled to prevent unwanted audio + processing during specific conversation states. + + Parameters: + FIRST_SPEECH: Mute STT until the first bot speech is detected. + MUTE_UNTIL_FIRST_BOT_COMPLETE: Mute STT until the first bot completes speaking, + regardless of whether it is the first speech. + FUNCTION_CALL: Mute STT during function calls to prevent interruptions. + ALWAYS: Always mute STT when the bot is speaking. + CUSTOM: Use a custom callback to determine muting logic dynamically. """ FIRST_SPEECH = "first_speech" @@ -58,10 +63,15 @@ class STTMuteStrategy(Enum): class STTMuteConfig: """Configuration for STT muting behavior. - Args: - strategies: Set of muting strategies to apply + Defines which muting strategies to apply and provides optional custom + callback for advanced muting logic. Multiple strategies can be combined + to create sophisticated muting behavior. + + Parameters: + strategies: Set of muting strategies to apply simultaneously. should_mute_callback: Optional callback for custom muting logic. - Only required when using STTMuteStrategy.CUSTOM + Only required when using STTMuteStrategy.CUSTOM. Called with + the STTMuteFilter instance to determine muting state. Note: MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together @@ -69,10 +79,14 @@ class STTMuteConfig: """ strategies: set[STTMuteStrategy] - # Optional callback for custom muting logic should_mute_callback: Optional[Callable[["STTMuteFilter"], Awaitable[bool]]] = None def __post_init__(self): + """Validate configuration after initialization. + + Raises: + ValueError: If incompatible strategies are used together. + """ if ( STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE in self.strategies and STTMuteStrategy.FIRST_SPEECH in self.strategies @@ -86,15 +100,18 @@ class STTMuteFilter(FrameProcessor): """A processor that handles STT muting and interruption control. This processor combines STT muting and interruption control as a coordinated - feature. When STT is muted, interruptions are automatically disabled. - - Args: - config: Configuration specifying muting strategies - stt_service: STT service instance (deprecated, will be removed in future version) - **kwargs: Additional arguments passed to parent class + feature. When STT is muted, interruptions are automatically disabled by + suppressing VAD-related frames. This prevents unwanted speech detection + during bot speech, function calls, or other specified conditions. """ def __init__(self, *, config: STTMuteConfig, **kwargs): + """Initialize the STT mute filter. + + Args: + config: Configuration specifying muting strategies and behavior. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._config = config self._first_speech_handled = False @@ -104,18 +121,22 @@ class STTMuteFilter(FrameProcessor): @property def is_muted(self) -> bool: - """Returns whether STT is currently muted.""" + """Check if STT is currently muted. + + Returns: + True if STT is currently muted and audio frames are being suppressed. + """ return self._is_muted async def _handle_mute_state(self, should_mute: bool): - """Handles both STT muting and interruption control.""" + """Handle STT muting and interruption control state changes.""" if should_mute != self.is_muted: logger.debug(f"STTMuteFilter {'muting' if should_mute else 'unmuting'}") self._is_muted = should_mute await self.push_frame(STTMuteFrame(mute=should_mute)) async def _should_mute(self) -> bool: - """Determines if STT should be muted based on current state and strategy.""" + """Determine if STT should be muted based on current state and strategies.""" for strategy in self._config.strategies: match strategy: case STTMuteStrategy.FUNCTION_CALL: @@ -144,7 +165,16 @@ class STTMuteFilter(FrameProcessor): return False async def process_frame(self, frame: Frame, direction: FrameDirection): - """Processes incoming frames and manages muting state.""" + """Process incoming frames and manage muting state. + + Monitors conversation state through frame types and applies muting + strategies accordingly. Suppresses VAD-related frames when muted + while allowing other frames to pass through. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) # Determine if we need to change mute state based on frame type diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index d8661eae1..1f01d0a1b 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Wake phrase detection filter for Pipecat transcription processing. + +This module provides a frame processor that filters transcription frames, +only allowing them through after wake phrases have been detected. Includes +keepalive functionality to maintain conversation flow after wake detection. +""" + import re import time from enum import Enum @@ -16,23 +23,53 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class WakeCheckFilter(FrameProcessor): - """This filter looks for wake phrases in the transcription frames and only passes through frames - after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief - period of continued conversation after a wake phrase has been detected. + """Frame processor that filters transcription frames based on wake phrase detection. + + This filter monitors transcription frames for configured wake phrases and only + passes frames through after a wake phrase has been detected. Maintains a + keepalive timeout to allow continued conversation after wake detection. """ class WakeState(Enum): + """Enumeration of wake detection states. + + Parameters: + IDLE: No wake phrase detected, filtering active. + AWAKE: Wake phrase detected, allowing frames through. + """ + IDLE = 1 AWAKE = 2 class ParticipantState: + """State tracking for individual participants. + + Parameters: + participant_id: Unique identifier for the participant. + state: Current wake state (IDLE or AWAKE). + wake_timer: Timestamp of last wake phrase detection. + accumulator: Accumulated text for wake phrase matching. + """ + def __init__(self, participant_id: str): + """Initialize participant state. + + Args: + participant_id: Unique identifier for the participant. + """ self.participant_id = participant_id self.state = WakeCheckFilter.WakeState.IDLE self.wake_timer = 0.0 self.accumulator = "" def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3): + """Initialize the wake phrase filter. + + Args: + wake_phrases: List of wake phrases to detect in transcriptions. + keepalive_timeout: Duration in seconds to keep passing frames after + wake detection. Defaults to 3 seconds. + """ super().__init__() self._participant_states = {} self._keepalive_timeout = keepalive_timeout @@ -44,6 +81,12 @@ class WakeCheckFilter(FrameProcessor): self._wake_patterns.append(pattern) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames, filtering transcriptions based on wake detection. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) try: diff --git a/src/pipecat/processors/filters/wake_notifier_filter.py b/src/pipecat/processors/filters/wake_notifier_filter.py index c7e6ed27a..c30f3b5d3 100644 --- a/src/pipecat/processors/filters/wake_notifier_filter.py +++ b/src/pipecat/processors/filters/wake_notifier_filter.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Wake notifier filter for conditional frame-based notifications.""" + from typing import Awaitable, Callable, Tuple, Type from pipecat.frames.frames import Frame @@ -12,10 +14,11 @@ from pipecat.sync.base_notifier import BaseNotifier class WakeNotifierFilter(FrameProcessor): - """This processor expects a list of frame types and will execute a given - callback predicate when a frame of any of those type is being processed. If - the callback returns true the notifier will be notified. + """Frame processor that conditionally triggers notifications based on frame types and filters. + This processor monitors frames of specified types and executes a callback predicate + when such frames are processed. If the callback returns True, the associated + notifier is triggered, allowing for conditional wake-up or notification scenarios. """ def __init__( @@ -26,12 +29,27 @@ class WakeNotifierFilter(FrameProcessor): filter: Callable[[Frame], Awaitable[bool]], **kwargs, ): + """Initialize the wake notifier filter. + + Args: + notifier: The notifier to trigger when conditions are met. + types: Tuple of frame types to monitor for potential notifications. + filter: Async callback that determines whether to trigger notification. + Should return True to trigger notification, False otherwise. + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) self._notifier = notifier self._types = types self._filter = filter async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and conditionally trigger notifications. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, self._types) and await self._filter(frame): diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 1935aeb2b..4105f4179 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame processing pipeline infrastructure for Pipecat. + +This module provides the core frame processing system that enables building +audio/video processing pipelines. It includes frame processors, pipeline +management, and frame flow control mechanisms. +""" + import asyncio from dataclasses import dataclass from enum import Enum @@ -36,12 +43,28 @@ from pipecat.utils.base_object import BaseObject class FrameDirection(Enum): + """Direction of frame flow in the processing pipeline. + + Parameters: + DOWNSTREAM: Frames flowing from input to output. + UPSTREAM: Frames flowing back from output to input. + """ + DOWNSTREAM = 1 UPSTREAM = 2 @dataclass class FrameProcessorSetup: + """Configuration parameters for frame processor initialization. + + Parameters: + clock: The clock instance for timing operations. + task_manager: The task manager for handling async operations. + observer: Optional observer for monitoring frame processing events. + watchdog_timers_enabled: Whether to enable watchdog timers by default. + """ + clock: BaseClock task_manager: BaseTaskManager observer: Optional[BaseObserver] = None @@ -49,6 +72,14 @@ class FrameProcessorSetup: class FrameProcessor(BaseObject): + """Base class for all frame processors in the pipeline. + + Frame processors are the building blocks of Pipecat pipelines. They receive + frames, process them, and pass them to the next processor in the chain. + Each processor runs in its own task and can be linked to form complex + processing pipelines. + """ + def __init__( self, *, @@ -59,6 +90,16 @@ class FrameProcessor(BaseObject): watchdog_timeout_secs: Optional[float] = None, **kwargs, ): + """Initialize the frame processor. + + Args: + name: Optional name for this processor instance. + enable_watchdog_logging: Whether to enable watchdog logging for tasks. + enable_watchdog_timers: Whether to enable watchdog timers for tasks. + metrics: Optional metrics collector for this processor. + watchdog_timeout_secs: Timeout in seconds for watchdog operations. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(name=name) self._parent: Optional["FrameProcessor"] = None self._prev: Optional["FrameProcessor"] = None @@ -118,77 +159,145 @@ class FrameProcessor(BaseObject): @property def id(self) -> int: + """Get the unique identifier for this processor. + + Returns: + The unique integer ID of this processor. + """ return self._id @property def name(self) -> str: + """Get the name of this processor. + + Returns: + The name of this processor instance. + """ return self._name @property def interruptions_allowed(self): + """Check if interruptions are allowed for this processor. + + Returns: + True if interruptions are allowed. + """ return self._allow_interruptions @property def metrics_enabled(self): + """Check if metrics collection is enabled. + + Returns: + True if metrics collection is enabled. + """ return self._enable_metrics @property def usage_metrics_enabled(self): + """Check if usage metrics collection is enabled. + + Returns: + True if usage metrics collection is enabled. + """ return self._enable_usage_metrics @property def report_only_initial_ttfb(self): + """Check if only initial TTFB should be reported. + + Returns: + True if only initial time-to-first-byte should be reported. + """ return self._report_only_initial_ttfb @property def interruption_strategies(self) -> Sequence[BaseInterruptionStrategy]: + """Get the interruption strategies for this processor. + + Returns: + Sequence of interruption strategies. + """ return self._interruption_strategies @property def task_manager(self) -> BaseTaskManager: + """Get the task manager for this processor. + + Returns: + The task manager instance. + + Raises: + Exception: If the task manager is not initialized. + """ if not self._task_manager: raise Exception(f"{self} TaskManager is still not initialized.") return self._task_manager def can_generate_metrics(self) -> bool: + """Check if this processor can generate metrics. + + Returns: + True if this processor can generate metrics. + """ return False def set_core_metrics_data(self, data: MetricsData): + """Set core metrics data for this processor. + + Args: + data: The metrics data to set. + """ self._metrics.set_core_metrics_data(data) async def start_ttfb_metrics(self): + """Start time-to-first-byte metrics collection.""" if self.can_generate_metrics() and self.metrics_enabled: await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb) async def stop_ttfb_metrics(self): + """Stop time-to-first-byte metrics collection and push results.""" if self.can_generate_metrics() and self.metrics_enabled: frame = await self._metrics.stop_ttfb_metrics() if frame: await self.push_frame(frame) async def start_processing_metrics(self): + """Start processing metrics collection.""" if self.can_generate_metrics() and self.metrics_enabled: await self._metrics.start_processing_metrics() async def stop_processing_metrics(self): + """Stop processing metrics collection and push results.""" if self.can_generate_metrics() and self.metrics_enabled: frame = await self._metrics.stop_processing_metrics() if frame: await self.push_frame(frame) async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): + """Start LLM usage metrics collection. + + Args: + tokens: Token usage information for the LLM. + """ if self.can_generate_metrics() and self.usage_metrics_enabled: frame = await self._metrics.start_llm_usage_metrics(tokens) if frame: await self.push_frame(frame) async def start_tts_usage_metrics(self, text: str): + """Start TTS usage metrics collection. + + Args: + text: The text being processed by TTS. + """ if self.can_generate_metrics() and self.usage_metrics_enabled: frame = await self._metrics.start_tts_usage_metrics(text) if frame: await self.push_frame(frame) async def stop_all_metrics(self): + """Stop all active metrics collection.""" await self.stop_ttfb_metrics() await self.stop_processing_metrics() @@ -201,6 +310,18 @@ class FrameProcessor(BaseObject): enable_watchdog_timers: Optional[bool] = None, watchdog_timeout_secs: Optional[float] = None, ) -> asyncio.Task: + """Create a new task managed by this processor. + + Args: + coroutine: The coroutine to run in the task. + name: Optional name for the task. + enable_watchdog_logging: Whether to enable watchdog logging. + enable_watchdog_timers: Whether to enable watchdog timers. + watchdog_timeout_secs: Timeout in seconds for watchdog operations. + + Returns: + The created asyncio task. + """ if name: name = f"{self}::{name}" else: @@ -222,15 +343,33 @@ class FrameProcessor(BaseObject): ) async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): + """Cancel a task managed by this processor. + + Args: + task: The task to cancel. + timeout: Optional timeout for task cancellation. + """ await self.task_manager.cancel_task(task, timeout) async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None): + """Wait for a task to complete. + + Args: + task: The task to wait for. + timeout: Optional timeout for waiting. + """ await self.task_manager.wait_for_task(task, timeout) def reset_watchdog(self): + """Reset the watchdog timer for the current task.""" self.task_manager.task_reset_watchdog() async def setup(self, setup: FrameProcessorSetup): + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ self._clock = setup.clock self._task_manager = setup.task_manager self._observer = setup.observer @@ -243,6 +382,7 @@ class FrameProcessor(BaseObject): await self._metrics.setup(self._task_manager) async def cleanup(self): + """Clean up processor resources.""" await super().cleanup() await self.__cancel_input_task() await self.__cancel_push_task() @@ -250,20 +390,48 @@ class FrameProcessor(BaseObject): await self._metrics.cleanup() def link(self, processor: "FrameProcessor"): + """Link this processor to the next processor in the pipeline. + + Args: + processor: The processor to link to. + """ self._next = processor processor._prev = self logger.debug(f"Linking {self} -> {self._next}") def get_event_loop(self) -> asyncio.AbstractEventLoop: + """Get the event loop used by this processor. + + Returns: + The asyncio event loop. + """ return self.task_manager.get_event_loop() def set_parent(self, parent: "FrameProcessor"): + """Set the parent processor for this processor. + + Args: + parent: The parent processor. + """ self._parent = parent def get_parent(self) -> Optional["FrameProcessor"]: + """Get the parent processor. + + Returns: + The parent processor, or None if no parent is set. + """ return self._parent def get_clock(self) -> BaseClock: + """Get the clock used by this processor. + + Returns: + The clock instance. + + Raises: + Exception: If the clock is not initialized. + """ if not self._clock: raise Exception(f"{self} Clock is still not initialized.") return self._clock @@ -276,6 +444,13 @@ class FrameProcessor(BaseObject): Callable[["FrameProcessor", Frame, FrameDirection], Awaitable[None]] ] = None, ): + """Queue a frame for processing. + + Args: + frame: The frame to queue. + direction: The direction of frame flow. + callback: Optional callback to call after processing. + """ # If we are cancelling we don't want to process any other frame. if self._cancelling: return @@ -288,15 +463,23 @@ class FrameProcessor(BaseObject): await self.__input_queue.put((frame, direction, callback)) async def pause_processing_frames(self): + """Pause processing of queued frames.""" logger.trace(f"{self}: pausing frame processing") self.__should_block_frames = True async def resume_processing_frames(self): + """Resume processing of queued frames.""" logger.trace(f"{self}: resuming frame processing") if self.__input_event: self.__input_event.set() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process a frame. + + Args: + frame: The frame to process. + direction: The direction of frame flow. + """ if isinstance(frame, StartFrame): await self.__start(frame) elif isinstance(frame, StartInterruptionFrame): @@ -312,9 +495,20 @@ class FrameProcessor(BaseObject): await self.__resume(frame) async def push_error(self, error: ErrorFrame): + """Push an error frame upstream. + + Args: + error: The error frame to push. + """ await self.push_frame(error, FrameDirection.UPSTREAM) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame to the next processor in the pipeline. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ if not self._check_started(frame): return @@ -324,6 +518,11 @@ class FrameProcessor(BaseObject): await self.__push_queue.put((frame, direction)) async def __start(self, frame: StartFrame): + """Handle the start frame to initialize processor state. + + Args: + frame: The start frame containing initialization parameters. + """ self.__started = True self._allow_interruptions = frame.allow_interruptions self._enable_metrics = frame.enable_metrics @@ -334,15 +533,30 @@ class FrameProcessor(BaseObject): self.__create_push_task() async def __cancel(self, frame: CancelFrame): + """Handle the cancel frame to stop processor operation. + + Args: + frame: The cancel frame. + """ self._cancelling = True await self.__cancel_input_task() await self.__cancel_push_task() async def __pause(self, frame: FrameProcessorPauseFrame | FrameProcessorPauseUrgentFrame): + """Handle pause frame to pause processor operation. + + Args: + frame: The pause frame. + """ if frame.processor.name == self.name: await self.pause_processing_frames() async def __resume(self, frame: FrameProcessorResumeFrame | FrameProcessorResumeUrgentFrame): + """Handle resume frame to resume processor operation. + + Args: + frame: The resume frame. + """ if frame.processor.name == self.name: await self.resume_processing_frames() @@ -351,6 +565,7 @@ class FrameProcessor(BaseObject): # async def _start_interruption(self): + """Start handling an interruption by canceling current tasks.""" try: # Cancel the push frame task. This will stop pushing frames downstream. await self.__cancel_push_task() @@ -368,10 +583,17 @@ class FrameProcessor(BaseObject): self.__create_push_task() async def _stop_interruption(self): + """Stop handling an interruption.""" # Nothing to do right now. pass async def __internal_push_frame(self, frame: Frame, direction: FrameDirection): + """Internal method to push frames to adjacent processors. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ try: timestamp = self._clock.get_time() if self._clock else 0 if direction == FrameDirection.DOWNSTREAM and self._next: @@ -404,11 +626,20 @@ class FrameProcessor(BaseObject): await self.push_error(ErrorFrame(str(e))) def _check_started(self, frame: Frame): + """Check if the processor has been started. + + Args: + frame: The frame being processed. + + Returns: + True if the processor has been started. + """ if not self.__started: logger.error(f"{self} Trying to process {frame} but StartFrame not received yet") return self.__started def __create_input_task(self): + """Create the input processing task.""" if not self.__input_frame_task: self.__should_block_frames = False if not self.__input_event: @@ -418,11 +649,13 @@ class FrameProcessor(BaseObject): self.__input_frame_task = self.create_task(self.__input_frame_task_handler()) async def __cancel_input_task(self): + """Cancel the input processing task.""" if self.__input_frame_task: await self.cancel_task(self.__input_frame_task) self.__input_frame_task = None async def __input_frame_task_handler(self): + """Handle frames from the input queue.""" while True: if self.__should_block_frames and self.__input_event: logger.trace(f"{self}: frame processing paused") @@ -445,16 +678,19 @@ class FrameProcessor(BaseObject): self.__input_queue.task_done() def __create_push_task(self): + """Create the frame pushing task.""" if not self.__push_frame_task: self.__push_queue = WatchdogQueue(self.task_manager) self.__push_frame_task = self.create_task(self.__push_frame_task_handler()) async def __cancel_push_task(self): + """Cancel the frame pushing task.""" if self.__push_frame_task: await self.cancel_task(self.__push_frame_task) self.__push_frame_task = None async def __push_frame_task_handler(self): + """Handle frames from the push queue.""" while True: (frame, direction) = await self.__push_queue.get() await self.__internal_push_frame(frame, direction) diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index dee197a51..65bf56b70 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Langchain integration processor for Pipecat.""" + from typing import Optional, Union from loguru import logger @@ -26,16 +28,40 @@ except ModuleNotFoundError as e: class LangchainProcessor(FrameProcessor): + """Processor that integrates Langchain runnables with Pipecat's frame pipeline. + + This processor takes LLM message frames, extracts the latest user message, + and processes it through a Langchain runnable chain. The response is streamed + back as text frames with appropriate response markers. + """ + def __init__(self, chain: Runnable, transcript_key: str = "input"): + """Initialize the Langchain processor. + + Args: + chain: The Langchain runnable to use for processing messages. + transcript_key: The key to use when passing input to the chain. + """ super().__init__() self._chain = chain self._transcript_key = transcript_key self._participant_id: Optional[str] = None def set_participant_id(self, participant_id: str): + """Set the participant ID for session tracking. + + Args: + participant_id: The participant ID to use for session configuration. + """ self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle LLM message frames. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, LLMMessagesFrame): @@ -50,6 +76,14 @@ class LangchainProcessor(FrameProcessor): @staticmethod def __get_token_value(text: Union[str, AIMessageChunk]) -> str: + """Extract token value from various text types. + + Args: + text: The text or message chunk to extract value from. + + Returns: + The extracted string value. + """ match text: case str(): return text @@ -59,6 +93,7 @@ class LangchainProcessor(FrameProcessor): return "" async def _ainvoke(self, text: str): + """Invoke the Langchain runnable with the provided text.""" logger.debug(f"Invoking chain with {text}") await self.push_frame(LLMFullResponseStartFrame()) try: diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 09291c422..22d5370f2 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""RTVI (Real-Time Voice Interface) protocol implementation for Pipecat. + +This module provides the RTVI protocol implementation for real-time voice interactions +between clients and AI agents. It includes message handling, action processing, +and frame observation for the RTVI protocol. +""" + import asyncio import base64 from dataclasses import dataclass @@ -79,6 +86,12 @@ ActionResult = Union[bool, int, float, str, list, dict] class RTVIServiceOption(BaseModel): + """Configuration option for an RTVI service. + + Defines a configurable option that can be set for an RTVI service, + including its name, type, and handler function. + """ + name: str type: Literal["bool", "number", "string", "array", "object"] handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"], Awaitable[None]] = Field( @@ -87,11 +100,18 @@ class RTVIServiceOption(BaseModel): class RTVIService(BaseModel): + """An RTVI service definition. + + Represents a service that can be configured and used within the RTVI protocol, + containing a name and list of configurable options. + """ + name: str options: List[RTVIServiceOption] _options_dict: Dict[str, RTVIServiceOption] = PrivateAttr(default={}) def model_post_init(self, __context: Any) -> None: + """Initialize the options dictionary after model creation.""" self._options_dict = {} for option in self.options: self._options_dict[option.name] = option @@ -99,16 +119,32 @@ class RTVIService(BaseModel): class RTVIActionArgumentData(BaseModel): + """Data for an RTVI action argument. + + Contains the name and value of an argument passed to an RTVI action. + """ + name: str value: Any class RTVIActionArgument(BaseModel): + """Definition of an RTVI action argument. + + Specifies the name and expected type of an argument for an RTVI action. + """ + name: str type: Literal["bool", "number", "string", "array", "object"] class RTVIAction(BaseModel): + """An RTVI action definition. + + Represents an action that can be executed within the RTVI protocol, + including its service, name, arguments, and handler function. + """ + service: str action: str arguments: List[RTVIActionArgument] = Field(default_factory=list) @@ -119,6 +155,7 @@ class RTVIAction(BaseModel): _arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={}) def model_post_init(self, __context: Any) -> None: + """Initialize the arguments dictionary after model creation.""" self._arguments_dict = {} for arg in self.arguments: self._arguments_dict[arg.name] = arg @@ -126,16 +163,31 @@ class RTVIAction(BaseModel): class RTVIServiceOptionConfig(BaseModel): + """Configuration value for an RTVI service option. + + Contains the name and value to set for a specific service option. + """ + name: str value: Any class RTVIServiceConfig(BaseModel): + """Configuration for an RTVI service. + + Contains the service name and list of option configurations to apply. + """ + service: str options: List[RTVIServiceOptionConfig] class RTVIConfig(BaseModel): + """Complete RTVI configuration. + + Contains the full configuration for all RTVI services. + """ + config: List[RTVIServiceConfig] @@ -145,16 +197,31 @@ class RTVIConfig(BaseModel): class RTVIUpdateConfig(BaseModel): + """Request to update RTVI configuration. + + Contains new configuration settings and whether to interrupt the bot. + """ + config: List[RTVIServiceConfig] interrupt: bool = False class RTVIActionRunArgument(BaseModel): + """Argument for running an RTVI action. + + Contains the name and value of an argument to pass to an action. + """ + name: str value: Any class RTVIActionRun(BaseModel): + """Request to run an RTVI action. + + Contains the service, action name, and optional arguments. + """ + service: str action: str arguments: Optional[List[RTVIActionRunArgument]] = None @@ -162,11 +229,23 @@ class RTVIActionRun(BaseModel): @dataclass class RTVIActionFrame(DataFrame): + """Frame containing an RTVI action to execute. + + Parameters: + rtvi_action_run: The action to execute. + message_id: Optional message ID for response correlation. + """ + rtvi_action_run: RTVIActionRun message_id: Optional[str] = None class RTVIMessage(BaseModel): + """Base RTVI message structure. + + Represents the standard format for RTVI protocol messages. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: str id: str @@ -179,10 +258,20 @@ class RTVIMessage(BaseModel): class RTVIErrorResponseData(BaseModel): + """Data for an RTVI error response. + + Contains the error message to send back to the client. + """ + error: str class RTVIErrorResponse(BaseModel): + """RTVI error response message. + + Sent in response to a client request that resulted in an error. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["error-response"] = "error-response" id: str @@ -190,21 +279,41 @@ class RTVIErrorResponse(BaseModel): class RTVIErrorData(BaseModel): + """Data for an RTVI error event. + + Contains error information including whether it's fatal. + """ + error: str fatal: bool class RTVIError(BaseModel): + """RTVI error event message. + + Sent when an error occurs that isn't in response to a specific request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["error"] = "error" data: RTVIErrorData class RTVIDescribeConfigData(BaseModel): + """Data for describing available RTVI configuration. + + Contains the list of available services and their options. + """ + config: List[RTVIService] class RTVIDescribeConfig(BaseModel): + """Message describing available RTVI configuration. + + Sent in response to a describe-config request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["config-available"] = "config-available" id: str @@ -212,10 +321,20 @@ class RTVIDescribeConfig(BaseModel): class RTVIDescribeActionsData(BaseModel): + """Data for describing available RTVI actions. + + Contains the list of available actions that can be executed. + """ + actions: List[RTVIAction] class RTVIDescribeActions(BaseModel): + """Message describing available RTVI actions. + + Sent in response to a describe-actions request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["actions-available"] = "actions-available" id: str @@ -223,6 +342,11 @@ class RTVIDescribeActions(BaseModel): class RTVIConfigResponse(BaseModel): + """Response containing current RTVI configuration. + + Sent in response to a get-config request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["config"] = "config" id: str @@ -230,10 +354,20 @@ class RTVIConfigResponse(BaseModel): class RTVIActionResponseData(BaseModel): + """Data for an RTVI action response. + + Contains the result of executing an action. + """ + result: ActionResult class RTVIActionResponse(BaseModel): + """Response to an RTVI action execution. + + Sent after successfully executing an action. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["action-response"] = "action-response" id: str @@ -241,11 +375,21 @@ class RTVIActionResponse(BaseModel): class RTVIBotReadyData(BaseModel): + """Data for bot ready notification. + + Contains protocol version and initial configuration. + """ + version: str config: List[RTVIServiceConfig] class RTVIBotReady(BaseModel): + """Message indicating bot is ready for interaction. + + Sent after bot initialization is complete. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-ready"] = "bot-ready" id: str @@ -253,28 +397,53 @@ class RTVIBotReady(BaseModel): class RTVILLMFunctionCallMessageData(BaseModel): + """Data for LLM function call notification. + + Contains function call details including name, ID, and arguments. + """ + function_name: str tool_call_id: str args: Mapping[str, Any] class RTVILLMFunctionCallMessage(BaseModel): + """Message notifying of an LLM function call. + + Sent when the LLM makes a function call. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["llm-function-call"] = "llm-function-call" data: RTVILLMFunctionCallMessageData class RTVILLMFunctionCallStartMessageData(BaseModel): + """Data for LLM function call start notification. + + Contains the function name being called. + """ + function_name: str class RTVILLMFunctionCallStartMessage(BaseModel): + """Message notifying that an LLM function call has started. + + Sent when the LLM begins a function call. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["llm-function-call-start"] = "llm-function-call-start" data: RTVILLMFunctionCallStartMessageData class RTVILLMFunctionCallResultData(BaseModel): + """Data for LLM function call result. + + Contains function call details and result. + """ + function_name: str tool_call_id: str arguments: dict @@ -282,60 +451,103 @@ class RTVILLMFunctionCallResultData(BaseModel): class RTVIBotLLMStartedMessage(BaseModel): + """Message indicating bot LLM processing has started.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-llm-started"] = "bot-llm-started" class RTVIBotLLMStoppedMessage(BaseModel): + """Message indicating bot LLM processing has stopped.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-llm-stopped"] = "bot-llm-stopped" class RTVIBotTTSStartedMessage(BaseModel): + """Message indicating bot TTS processing has started.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-started"] = "bot-tts-started" class RTVIBotTTSStoppedMessage(BaseModel): + """Message indicating bot TTS processing has stopped.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-stopped"] = "bot-tts-stopped" class RTVITextMessageData(BaseModel): + """Data for text-based RTVI messages. + + Contains text content. + """ + text: str class RTVIBotTranscriptionMessage(BaseModel): + """Message containing bot transcription text. + + Sent when the bot's speech is transcribed. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-transcription"] = "bot-transcription" data: RTVITextMessageData class RTVIBotLLMTextMessage(BaseModel): + """Message containing bot LLM text output. + + Sent when the bot's LLM generates text. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-llm-text"] = "bot-llm-text" data: RTVITextMessageData class RTVIBotTTSTextMessage(BaseModel): + """Message containing bot TTS text output. + + Sent when text is being processed by TTS. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-text"] = "bot-tts-text" data: RTVITextMessageData class RTVIAudioMessageData(BaseModel): + """Data for audio-based RTVI messages. + + Contains audio data and metadata. + """ + audio: str sample_rate: int num_channels: int class RTVIBotTTSAudioMessage(BaseModel): + """Message containing bot TTS audio output. + + Sent when the bot's TTS generates audio. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-audio"] = "bot-tts-audio" data: RTVIAudioMessageData class RTVIUserTranscriptionMessageData(BaseModel): + """Data for user transcription messages. + + Contains transcription text and metadata. + """ + text: str user_id: str timestamp: str @@ -343,44 +555,72 @@ class RTVIUserTranscriptionMessageData(BaseModel): class RTVIUserTranscriptionMessage(BaseModel): + """Message containing user transcription. + + Sent when user speech is transcribed. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-transcription"] = "user-transcription" data: RTVIUserTranscriptionMessageData class RTVIUserLLMTextMessage(BaseModel): + """Message containing user text input for LLM. + + Sent when user text is processed by the LLM. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-llm-text"] = "user-llm-text" data: RTVITextMessageData class RTVIUserStartedSpeakingMessage(BaseModel): + """Message indicating user has started speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-started-speaking"] = "user-started-speaking" class RTVIUserStoppedSpeakingMessage(BaseModel): + """Message indicating user has stopped speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-stopped-speaking"] = "user-stopped-speaking" class RTVIBotStartedSpeakingMessage(BaseModel): + """Message indicating bot has started speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-started-speaking"] = "bot-started-speaking" class RTVIBotStoppedSpeakingMessage(BaseModel): + """Message indicating bot has stopped speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-stopped-speaking"] = "bot-stopped-speaking" class RTVIMetricsMessage(BaseModel): + """Message containing performance metrics. + + Sent to provide performance and usage metrics. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["metrics"] = "metrics" data: Mapping[str, Any] class RTVIServerMessage(BaseModel): + """Generic server message. + + Used for custom server-to-client messages. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["server-message"] = "server-message" data: Any @@ -388,28 +628,32 @@ class RTVIServerMessage(BaseModel): @dataclass class RTVIServerMessageFrame(SystemFrame): - """A frame for sending server messages to the client.""" + """A frame for sending server messages to the client. + + Parameters: + data: The message data to send to the client. + """ data: Any def __str__(self): + """String representation of the RTVI server message frame.""" return f"{self.name}(data: {self.data})" @dataclass class RTVIObserverParams: - """ - Parameters for configuring RTVI Observer behavior. + """Parameters for configuring RTVI Observer behavior. - Attributes: - bot_llm_enabled (bool): Indicates if the bot's LLM messages should be sent. - bot_tts_enabled (bool): Indicates if the bot's TTS messages should be sent. - bot_speaking_enabled (bool): Indicates if the bot's started/stopped speaking messages should be sent. - user_llm_enabled (bool): Indicates if the user's LLM input messages should be sent. - user_speaking_enabled (bool): Indicates if the user's started/stopped speaking messages should be sent. - user_transcription_enabled (bool): Indicates if user's transcription messages should be sent. - metrics_enabled (bool): Indicates if metrics messages should be sent. - errors_enabled (bool): Indicates if errors messages should be sent. + Parameters: + bot_llm_enabled: Indicates if the bot's LLM messages should be sent. + bot_tts_enabled: Indicates if the bot's TTS messages should be sent. + bot_speaking_enabled: Indicates if the bot's started/stopped speaking messages should be sent. + user_llm_enabled: Indicates if the user's LLM input messages should be sent. + user_speaking_enabled: Indicates if the user's started/stopped speaking messages should be sent. + user_transcription_enabled: Indicates if user's transcription messages should be sent. + metrics_enabled: Indicates if metrics messages should be sent. + errors_enabled: Indicates if errors messages should be sent. """ bot_llm_enabled: bool = True @@ -432,15 +676,18 @@ class RTVIObserver(BaseObserver): Note: This observer only handles outgoing messages. Incoming RTVI client messages are handled by the RTVIProcessor. - - Args: - rtvi (RTVIProcessor): The RTVI processor to push frames to. - params (RTVIObserverParams): Settings to enable/disable specific messages. """ def __init__( self, rtvi: "RTVIProcessor", *, params: Optional[RTVIObserverParams] = None, **kwargs ): + """Initialize the RTVI observer. + + Args: + rtvi: The RTVI processor to push frames to. + params: Settings to enable/disable specific messages. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._rtvi = rtvi self._params = params or RTVIObserverParams() @@ -452,11 +699,7 @@ class RTVIObserver(BaseObserver): """Process a frame being pushed through the pipeline. Args: - src: Source processor pushing the frame - dst: Destination processor receiving the frame - frame: The frame being pushed - direction: Direction of frame flow in pipeline - timestamp: Time when frame was pushed + data: Frame push event data containing source, frame, direction, and timestamp. """ src = data.source frame = data.frame @@ -517,13 +760,14 @@ class RTVIObserver(BaseObserver): """Push an urgent transport message to the RTVI processor. Args: - model: The message model to send - exclude_none: Whether to exclude None values from the model dump + model: The message model to send. + exclude_none: Whether to exclude None values from the model dump. """ frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) await self._rtvi.push_frame(frame) async def _push_bot_transcription(self): + """Push accumulated bot transcription as a message.""" if len(self._bot_transcription) > 0: message = RTVIBotTranscriptionMessage( data=RTVITextMessageData(text=self._bot_transcription) @@ -532,6 +776,7 @@ class RTVIObserver(BaseObserver): self._bot_transcription = "" async def _handle_interruptions(self, frame: Frame): + """Handle user speaking interruption frames.""" message = None if isinstance(frame, UserStartedSpeakingFrame): message = RTVIUserStartedSpeakingMessage() @@ -542,6 +787,7 @@ class RTVIObserver(BaseObserver): await self.push_transport_message_urgent(message) async def _handle_bot_speaking(self, frame: Frame): + """Handle bot speaking event frames.""" message = None if isinstance(frame, BotStartedSpeakingFrame): message = RTVIBotStartedSpeakingMessage() @@ -552,6 +798,7 @@ class RTVIObserver(BaseObserver): await self.push_transport_message_urgent(message) async def _handle_llm_text_frame(self, frame: LLMTextFrame): + """Handle LLM text output frames.""" message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) await self.push_transport_message_urgent(message) @@ -560,6 +807,7 @@ class RTVIObserver(BaseObserver): await self._push_bot_transcription() async def _handle_user_transcriptions(self, frame: Frame): + """Handle user transcription frames.""" message = None if isinstance(frame, TranscriptionFrame): message = RTVIUserTranscriptionMessage( @@ -608,6 +856,7 @@ class RTVIObserver(BaseObserver): logger.warning(f"Caught an error while trying to handle context: {e}") async def _handle_metrics(self, frame: MetricsFrame): + """Handle metrics frames and convert to RTVI metrics messages.""" metrics = {} for d in frame.data: if isinstance(d, TTFBMetricsData): @@ -632,6 +881,13 @@ class RTVIObserver(BaseObserver): class RTVIProcessor(FrameProcessor): + """Main processor for handling RTVI protocol messages and actions. + + This processor manages the RTVI protocol communication including client-server + handshaking, configuration management, action execution, and message routing. + It serves as the central hub for RTVI protocol operations. + """ + def __init__( self, *, @@ -639,6 +895,13 @@ class RTVIProcessor(FrameProcessor): transport: Optional[BaseTransport] = None, **kwargs, ): + """Initialize the RTVI processor. + + Args: + config: Initial RTVI configuration. + transport: Transport layer for communication. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._config = config or RTVIConfig(config=[]) @@ -668,34 +931,67 @@ class RTVIProcessor(FrameProcessor): self._input_transport.enable_audio_in_stream_on_start(False) def register_action(self, action: RTVIAction): + """Register an action that can be executed via RTVI. + + Args: + action: The action to register. + """ id = self._action_id(action.service, action.action) self._registered_actions[id] = action def register_service(self, service: RTVIService): + """Register a service that can be configured via RTVI. + + Args: + service: The service to register. + """ self._registered_services[service.name] = service async def set_client_ready(self): + """Mark the client as ready and trigger the ready event.""" self._client_ready = True await self._call_event_handler("on_client_ready") async def set_bot_ready(self): + """Mark the bot as ready and send the bot-ready message.""" self._bot_ready = True await self._update_config(self._config, False) await self._send_bot_ready() def set_errors_enabled(self, enabled: bool): + """Enable or disable error message sending. + + Args: + enabled: Whether to send error messages. + """ self._errors_enabled = enabled async def interrupt_bot(self): + """Send a bot interruption frame upstream.""" await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) async def send_error(self, error: str): + """Send an error message to the client. + + Args: + error: The error message to send. + """ await self._send_error_frame(ErrorFrame(error=error)) async def handle_message(self, message: RTVIMessage): + """Handle an incoming RTVI message. + + Args: + message: The RTVI message to handle. + """ await self._message_queue.put(message) async def handle_function_call(self, params: FunctionCallParams): + """Handle a function call from the LLM. + + Args: + params: The function call parameters. + """ fn = RTVILLMFunctionCallMessageData( function_name=params.function_name, tool_call_id=params.tool_call_id, @@ -707,6 +1003,16 @@ class RTVIProcessor(FrameProcessor): async def handle_function_call_start( self, function_name: str, llm: FrameProcessor, context: OpenAILLMContext ): + """Handle the start of a function call from the LLM. + + Args: + function_name: Name of the function being called. + llm: The LLM processor making the call. + context: The LLM context. + + Note: + This method is deprecated. Use handle_function_call() instead. + """ import warnings with warnings.catch_warnings(): @@ -721,6 +1027,12 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message, exclude_none=False) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames through the RTVI processor. + + Args: + frame: The frame to process. + direction: The direction of frame flow. + """ await super().process_frame(frame, direction) # Specific system frames @@ -754,6 +1066,7 @@ class RTVIProcessor(FrameProcessor): await self.push_frame(frame, direction) async def _start(self, frame: StartFrame): + """Start the RTVI processor tasks.""" if not self._action_task: self._action_queue = WatchdogQueue(self.task_manager) self._action_task = self.create_task(self._action_task_handler()) @@ -763,12 +1076,15 @@ class RTVIProcessor(FrameProcessor): await self._call_event_handler("on_bot_started") async def _stop(self, frame: EndFrame): + """Stop the RTVI processor tasks.""" await self._cancel_tasks() async def _cancel(self, frame: CancelFrame): + """Cancel the RTVI processor tasks.""" await self._cancel_tasks() async def _cancel_tasks(self): + """Cancel all running tasks.""" if self._action_task: await self.cancel_task(self._action_task) self._action_task = None @@ -778,22 +1094,26 @@ class RTVIProcessor(FrameProcessor): self._message_task = None async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True): + """Push a transport message frame.""" frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) await self.push_frame(frame) async def _action_task_handler(self): + """Handle incoming action frames.""" while True: frame = await self._action_queue.get() await self._handle_action(frame.message_id, frame.rtvi_action_run) self._action_queue.task_done() async def _message_task_handler(self): + """Handle incoming transport messages.""" while True: message = await self._message_queue.get() await self._handle_message(message) self._message_queue.task_done() async def _handle_transport_message(self, frame: TransportMessageUrgentFrame): + """Handle an incoming transport message frame.""" try: transport_message = frame.message if transport_message.get("label") != RTVI_MESSAGE_LABEL: @@ -806,6 +1126,7 @@ class RTVIProcessor(FrameProcessor): logger.warning(f"Invalid RTVI transport message: {e}") async def _handle_message(self, message: RTVIMessage): + """Handle a parsed RTVI message.""" try: match message.type: case "client-ready": @@ -842,6 +1163,7 @@ class RTVIProcessor(FrameProcessor): logger.warning(f"Exception processing message: {e}") async def _handle_client_ready(self, request_id: str): + """Handle a client-ready message.""" logger.debug("Received client-ready") if self._input_transport: await self._input_transport.start_audio_in_streaming() @@ -850,6 +1172,7 @@ class RTVIProcessor(FrameProcessor): await self.set_client_ready() async def _handle_audio_buffer(self, data): + """Handle incoming audio buffer data.""" if not self._input_transport: return @@ -871,20 +1194,24 @@ class RTVIProcessor(FrameProcessor): logger.error(f"Error processing audio buffer: {e}") async def _handle_describe_config(self, request_id: str): + """Handle a describe-config request.""" services = list(self._registered_services.values()) message = RTVIDescribeConfig(id=request_id, data=RTVIDescribeConfigData(config=services)) await self._push_transport_message(message) async def _handle_describe_actions(self, request_id: str): + """Handle a describe-actions request.""" actions = list(self._registered_actions.values()) message = RTVIDescribeActions(id=request_id, data=RTVIDescribeActionsData(actions=actions)) await self._push_transport_message(message) async def _handle_get_config(self, request_id: str): + """Handle a get-config request.""" message = RTVIConfigResponse(id=request_id, data=self._config) await self._push_transport_message(message) def _update_config_option(self, service: str, config: RTVIServiceOptionConfig): + """Update a specific configuration option.""" for service_config in self._config.config: if service_config.service == service: for option_config in service_config.options: @@ -896,6 +1223,7 @@ class RTVIProcessor(FrameProcessor): service_config.options.append(config) async def _update_service_config(self, config: RTVIServiceConfig): + """Update configuration for a specific service.""" service = self._registered_services[config.service] for option in config.options: handler = service._options_dict[option.name].handler @@ -903,16 +1231,19 @@ class RTVIProcessor(FrameProcessor): self._update_config_option(service.name, option) async def _update_config(self, data: RTVIConfig, interrupt: bool): + """Update the RTVI configuration.""" if interrupt: await self.interrupt_bot() for service_config in data.config: await self._update_service_config(service_config) async def _handle_update_config(self, request_id: str, data: RTVIUpdateConfig): + """Handle an update-config request.""" await self._update_config(RTVIConfig(config=data.config), data.interrupt) await self._handle_get_config(request_id) async def _handle_function_call_result(self, data): + """Handle a function call result from the client.""" frame = FunctionCallResultFrame( function_name=data.function_name, tool_call_id=data.tool_call_id, @@ -922,6 +1253,7 @@ class RTVIProcessor(FrameProcessor): await self.push_frame(frame) async def _handle_action(self, request_id: Optional[str], data: RTVIActionRun): + """Handle an action execution request.""" action_id = self._action_id(data.service, data.action) if action_id not in self._registered_actions: await self._send_error_response(request_id, f"Action {action_id} not registered") @@ -939,6 +1271,7 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message) async def _send_bot_ready(self): + """Send the bot-ready message to the client.""" message = RTVIBotReady( id=self._client_ready_id, data=RTVIBotReadyData(version=RTVI_PROTOCOL_VERSION, config=self._config.config), @@ -946,14 +1279,17 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message) async def _send_error_frame(self, frame: ErrorFrame): + """Send an error frame as an RTVI error message.""" if self._errors_enabled: message = RTVIError(data=RTVIErrorData(error=frame.error, fatal=frame.fatal)) await self._push_transport_message(message) async def _send_error_response(self, id: str, error: str): + """Send an error response message.""" if self._errors_enabled: message = RTVIErrorResponse(id=id, data=RTVIErrorResponseData(error=error)) await self._push_transport_message(message) def _action_id(self, service: str, action: str) -> str: + """Generate an action ID from service and action names.""" return f"{service}:{action}" diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 84b8a050b..15f483029 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""GStreamer pipeline source integration for Pipecat.""" + import asyncio from typing import Optional @@ -36,7 +38,24 @@ except ModuleNotFoundError as e: class GStreamerPipelineSource(FrameProcessor): + """A frame processor that uses GStreamer pipelines as media sources. + + This processor creates and manages GStreamer pipelines to generate audio and video + output frames. It handles pipeline lifecycle, decoding, format conversion, and + frame generation with configurable output parameters. + """ + class OutputParams(BaseModel): + """Output configuration parameters for GStreamer pipeline. + + Parameters: + video_width: Width of output video frames in pixels. + video_height: Height of output video frames in pixels. + audio_sample_rate: Sample rate for audio output. If None, uses frame sample rate. + audio_channels: Number of audio channels for output. + clock_sync: Whether to synchronize output with pipeline clock. + """ + video_width: int = 1280 video_height: int = 720 audio_sample_rate: Optional[int] = None @@ -44,6 +63,13 @@ class GStreamerPipelineSource(FrameProcessor): clock_sync: bool = True def __init__(self, *, pipeline: str, out_params: Optional[OutputParams] = None, **kwargs): + """Initialize the GStreamer pipeline source. + + Args: + pipeline: GStreamer pipeline description string for the source. + out_params: Output configuration parameters. If None, uses defaults. + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) self._out_params = out_params or GStreamerPipelineSource.OutputParams() @@ -67,6 +93,12 @@ class GStreamerPipelineSource(FrameProcessor): bus.connect("message", self._on_gstreamer_message) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and manage GStreamer pipeline lifecycle. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) # Specific system frames @@ -92,13 +124,16 @@ class GStreamerPipelineSource(FrameProcessor): await self.push_frame(frame, direction) async def _start(self, frame: StartFrame): + """Start the GStreamer pipeline.""" self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate self._player.set_state(Gst.State.PLAYING) async def _stop(self, frame: EndFrame): + """Stop the GStreamer pipeline.""" self._player.set_state(Gst.State.NULL) async def _cancel(self, frame: CancelFrame): + """Cancel the GStreamer pipeline.""" self._player.set_state(Gst.State.NULL) # @@ -106,6 +141,7 @@ class GStreamerPipelineSource(FrameProcessor): # def _on_gstreamer_message(self, bus: Gst.Bus, message: Gst.Message): + """Handle GStreamer bus messages.""" t = message.type if t == Gst.MessageType.ERROR: err, debug = message.parse_error() @@ -113,6 +149,7 @@ class GStreamerPipelineSource(FrameProcessor): return True def _decodebin_callback(self, decodebin: Gst.Element, pad: Gst.Pad): + """Handle new pads from decodebin element.""" caps_string = pad.get_current_caps().to_string() if caps_string.startswith("audio"): self._decodebin_audio(pad) @@ -120,6 +157,7 @@ class GStreamerPipelineSource(FrameProcessor): self._decodebin_video(pad) def _decodebin_audio(self, pad: Gst.Pad): + """Set up audio processing pipeline from decoded audio pad.""" queue_audio = Gst.ElementFactory.make("queue", None) audioconvert = Gst.ElementFactory.make("audioconvert", None) audioresample = Gst.ElementFactory.make("audioresample", None) @@ -153,6 +191,7 @@ class GStreamerPipelineSource(FrameProcessor): pad.link(queue_pad) def _decodebin_video(self, pad: Gst.Pad): + """Set up video processing pipeline from decoded video pad.""" queue_video = Gst.ElementFactory.make("queue", None) videoconvert = Gst.ElementFactory.make("videoconvert", None) videoscale = Gst.ElementFactory.make("videoscale", None) @@ -187,6 +226,7 @@ class GStreamerPipelineSource(FrameProcessor): pad.link(queue_pad) def _appsink_audio_new_sample(self, appsink: GstApp.AppSink): + """Handle new audio samples from GStreamer appsink.""" buffer = appsink.pull_sample().get_buffer() (_, info) = buffer.map(Gst.MapFlags.READ) frame = OutputAudioRawFrame( @@ -199,6 +239,7 @@ class GStreamerPipelineSource(FrameProcessor): return Gst.FlowReturn.OK def _appsink_video_new_sample(self, appsink: GstApp.AppSink): + """Handle new video samples from GStreamer appsink.""" buffer = appsink.pull_sample().get_buffer() (_, info) = buffer.map(Gst.MapFlags.READ) frame = OutputImageRawFrame( diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index 36c7e9821..672027491 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Idle frame processor for timeout-based callback execution.""" + import asyncio from typing import Awaitable, Callable, List, Optional @@ -12,9 +14,11 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class IdleFrameProcessor(FrameProcessor): - """This class waits to receive any frame or list of desired frames within a - given timeout. If the timeout is reached before receiving any of those - frames the provided callback will be called. + """Monitors frame activity and triggers callbacks on timeout. + + This processor waits to receive any frame or specific frame types within a + given timeout period. If the timeout is reached before receiving the expected + frames, the provided callback will be executed. """ def __init__( @@ -25,6 +29,16 @@ class IdleFrameProcessor(FrameProcessor): types: Optional[List[type]] = None, **kwargs, ): + """Initialize the idle frame processor. + + Args: + callback: Async callback function to execute on timeout. Receives + this processor instance as an argument. + timeout: Timeout duration in seconds before triggering the callback. + types: Optional list of frame types to monitor. If None, monitors + all frames. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._callback = callback @@ -33,6 +47,12 @@ class IdleFrameProcessor(FrameProcessor): self._idle_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and manage idle timeout monitoring. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -50,15 +70,18 @@ class IdleFrameProcessor(FrameProcessor): self._idle_event.set() async def cleanup(self): + """Clean up resources and cancel pending tasks.""" if self._idle_task: await self.cancel_task(self._idle_task) def _create_idle_task(self): + """Create and start the idle monitoring task.""" if not self._idle_task: self._idle_event = asyncio.Event() self._idle_task = self.create_task(self._idle_task_handler()) async def _idle_task_handler(self): + """Handle idle timeout monitoring and callback execution.""" while True: try: await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout) diff --git a/src/pipecat/processors/logger.py b/src/pipecat/processors/logger.py index b773df8c3..acc85c40b 100644 --- a/src/pipecat/processors/logger.py +++ b/src/pipecat/processors/logger.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame logging utilities for debugging and monitoring frame flow in Pipecat pipelines.""" + from typing import Optional, Tuple, Type from loguru import logger @@ -21,6 +23,13 @@ logger = logger.opt(ansi=True) class FrameLogger(FrameProcessor): + """A frame processor that logs frame information for debugging purposes. + + This processor intercepts frames passing through the pipeline and logs + their details with configurable formatting and filtering. Useful for + debugging frame flow and understanding pipeline behavior. + """ + def __init__( self, prefix="Frame", @@ -32,12 +41,26 @@ class FrameLogger(FrameProcessor): TransportMessageFrame, ), ): + """Initialize the frame logger. + + Args: + prefix: Text prefix to add to log messages. Defaults to "Frame". + color: ANSI color code for log message formatting. If None, no coloring is applied. + ignored_frame_types: Tuple of frame types to exclude from logging. + Defaults to common high-frequency frames like audio and speaking frames. + """ super().__init__() self._prefix = prefix self._color = color self._ignored_frame_types = ignored_frame_types async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process and log frame information. + + Args: + frame: The frame to process and potentially log. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if self._ignored_frame_types and not isinstance(frame, self._ignored_frame_types): diff --git a/src/pipecat/processors/metrics/frame_processor_metrics.py b/src/pipecat/processors/metrics/frame_processor_metrics.py index cf08f85f6..386164afe 100644 --- a/src/pipecat/processors/metrics/frame_processor_metrics.py +++ b/src/pipecat/processors/metrics/frame_processor_metrics.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame processor metrics collection and reporting.""" + import time from typing import Optional @@ -23,7 +25,20 @@ from pipecat.utils.base_object import BaseObject class FrameProcessorMetrics(BaseObject): + """Metrics collection and reporting for frame processors. + + Provides comprehensive metrics tracking for frame processing operations, + including timing measurements, resource usage, and performance analytics. + Supports TTFB tracking, processing duration metrics, and usage statistics + for LLM and TTS operations. + """ + def __init__(self): + """Initialize the frame processor metrics collector. + + Sets up internal state for tracking various metrics including TTFB, + processing times, and usage statistics. + """ super().__init__() self._task_manager = None self._start_ttfb_time = 0 @@ -32,13 +47,24 @@ class FrameProcessorMetrics(BaseObject): self._should_report_ttfb = True async def setup(self, task_manager: BaseTaskManager): + """Set up the metrics collector with a task manager. + + Args: + task_manager: The task manager for handling async operations. + """ self._task_manager = task_manager async def cleanup(self): + """Clean up metrics collection resources.""" await super().cleanup() @property def task_manager(self) -> BaseTaskManager: + """Get the associated task manager. + + Returns: + The task manager instance for async operations. + """ return self._task_manager @property @@ -46,7 +72,7 @@ class FrameProcessorMetrics(BaseObject): """Get the current TTFB value in seconds. Returns: - Optional[float]: The TTFB value in seconds, or None if not measured + The TTFB value in seconds, or None if not measured. """ if self._last_ttfb_time > 0: return self._last_ttfb_time @@ -58,24 +84,46 @@ class FrameProcessorMetrics(BaseObject): return None def _processor_name(self): + """Get the processor name from core metrics data.""" return self._core_metrics_data.processor def _model_name(self): + """Get the model name from core metrics data.""" return self._core_metrics_data.model def set_core_metrics_data(self, data: MetricsData): + """Set the core metrics data for this collector. + + Args: + data: The core metrics data containing processor and model information. + """ self._core_metrics_data = data def set_processor_name(self, name: str): + """Set the processor name for metrics reporting. + + Args: + name: The name of the processor to use in metrics. + """ self._core_metrics_data = MetricsData(processor=name) async def start_ttfb_metrics(self, report_only_initial_ttfb): + """Start measuring time-to-first-byte (TTFB). + + Args: + report_only_initial_ttfb: Whether to report only the first TTFB measurement. + """ if self._should_report_ttfb: self._start_ttfb_time = time.time() self._last_ttfb_time = 0 self._should_report_ttfb = not report_only_initial_ttfb async def stop_ttfb_metrics(self): + """Stop TTFB measurement and generate metrics frame. + + Returns: + MetricsFrame containing TTFB data, or None if not measuring. + """ if self._start_ttfb_time == 0: return None @@ -88,9 +136,15 @@ class FrameProcessorMetrics(BaseObject): return MetricsFrame(data=[ttfb]) async def start_processing_metrics(self): + """Start measuring processing time.""" self._start_processing_time = time.time() async def stop_processing_metrics(self): + """Stop processing time measurement and generate metrics frame. + + Returns: + MetricsFrame containing processing duration data, or None if not measuring. + """ if self._start_processing_time == 0: return None @@ -103,6 +157,14 @@ class FrameProcessorMetrics(BaseObject): return MetricsFrame(data=[processing]) async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): + """Record LLM token usage metrics. + + Args: + tokens: Token usage information including prompt and completion tokens. + + Returns: + MetricsFrame containing LLM usage data. + """ logger.debug( f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}" ) @@ -112,6 +174,14 @@ class FrameProcessorMetrics(BaseObject): return MetricsFrame(data=[value]) async def start_tts_usage_metrics(self, text: str): + """Record TTS character usage metrics. + + Args: + text: The text being processed by TTS. + + Returns: + MetricsFrame containing TTS usage data. + """ characters = TTSUsageMetricsData( processor=self._processor_name(), model=self._model_name(), value=len(text) ) diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index 32b04a59b..755d5cd5e 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio +"""Sentry integration for frame processor metrics.""" from loguru import logger @@ -22,7 +22,19 @@ from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMet class SentryMetrics(FrameProcessorMetrics): + """Frame processor metrics integration with Sentry monitoring. + + Extends FrameProcessorMetrics to send time-to-first-byte (TTFB) and + processing metrics as Sentry transactions for performance monitoring + and debugging. + """ + def __init__(self): + """Initialize the Sentry metrics collector. + + Sets up internal state for tracking transactions and verifies + Sentry SDK initialization status. + """ super().__init__() self._ttfb_metrics_tx = None self._processing_metrics_tx = None @@ -32,6 +44,11 @@ class SentryMetrics(FrameProcessorMetrics): self._sentry_task = None async def setup(self, task_manager: BaseTaskManager): + """Setup the Sentry metrics system. + + Args: + task_manager: The task manager to use for background operations. + """ await super().setup(task_manager) if self._sentry_available: self._sentry_queue = WatchdogQueue(task_manager) @@ -40,6 +57,10 @@ class SentryMetrics(FrameProcessorMetrics): ) async def cleanup(self): + """Clean up Sentry resources and flush pending transactions. + + Ensures all pending transactions are sent to Sentry before shutdown. + """ await super().cleanup() if self._sentry_task: await self._sentry_queue.put(None) @@ -49,6 +70,11 @@ class SentryMetrics(FrameProcessorMetrics): sentry_sdk.flush(timeout=5.0) async def start_ttfb_metrics(self, report_only_initial_ttfb): + """Start tracking time-to-first-byte metrics. + + Args: + report_only_initial_ttfb: Whether to report only the initial TTFB measurement. + """ await super().start_ttfb_metrics(report_only_initial_ttfb) if self._should_report_ttfb and self._sentry_available: @@ -61,6 +87,10 @@ class SentryMetrics(FrameProcessorMetrics): ) async def stop_ttfb_metrics(self): + """Stop tracking time-to-first-byte metrics. + + Queues the TTFB transaction for completion and transmission to Sentry. + """ await super().stop_ttfb_metrics() if self._sentry_available and self._ttfb_metrics_tx: @@ -68,6 +98,10 @@ class SentryMetrics(FrameProcessorMetrics): self._ttfb_metrics_tx = None async def start_processing_metrics(self): + """Start tracking frame processing metrics. + + Creates a new Sentry transaction to track processing performance. + """ await super().start_processing_metrics() if self._sentry_available: @@ -80,6 +114,10 @@ class SentryMetrics(FrameProcessorMetrics): ) async def stop_processing_metrics(self): + """Stop tracking frame processing metrics. + + Queues the processing transaction for completion and transmission to Sentry. + """ await super().stop_processing_metrics() if self._sentry_available and self._processing_metrics_tx: @@ -87,6 +125,7 @@ class SentryMetrics(FrameProcessorMetrics): self._processing_metrics_tx = None async def _sentry_task_handler(self): + """Background task handler for completing Sentry transactions.""" running = True while running: tx = await self._sentry_queue.get() diff --git a/src/pipecat/processors/producer_processor.py b/src/pipecat/processors/producer_processor.py index 0a41269fb..8de9d66bb 100644 --- a/src/pipecat/processors/producer_processor.py +++ b/src/pipecat/processors/producer_processor.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Producer processor for frame filtering and distribution.""" + import asyncio from typing import Awaitable, Callable, List @@ -13,15 +15,24 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue async def identity_transformer(frame: Frame): + """Default transformer that returns the frame unchanged. + + Args: + frame: The frame to transform. + + Returns: + The same frame without modifications. + """ return frame class ProducerProcessor(FrameProcessor): - """This class optionally passes-through received frames and decides if those - frames should be sent to consumers based on a user-defined filter. The - frames can be transformed into a different type of frame before being - sending them to the consumers. More than one consumer can be added. + """A processor that filters frames and distributes them to multiple consumers. + This processor receives frames, applies a filter to determine which frames + should be sent to consumers (ConsumerProcessor), optionally transforms those + frames, and distributes them to registered consumer queues. It can also pass + frames through to the next processor in the pipeline. """ def __init__( @@ -31,6 +42,16 @@ class ProducerProcessor(FrameProcessor): transformer: Callable[[Frame], Awaitable[Frame]] = identity_transformer, passthrough: bool = True, ): + """Initialize the producer processor. + + Args: + filter: Async function that determines if a frame should be produced. + Must return True for frames to be sent to consumers. + transformer: Async function to transform frames before sending to consumers. + Defaults to identity_transformer which returns frames unchanged. + passthrough: Whether to pass frames through to the next processor. + If True, all frames continue downstream regardless of filter result. + """ super().__init__() self._filter = filter self._transformer = transformer @@ -38,8 +59,7 @@ class ProducerProcessor(FrameProcessor): self._consumers: List[asyncio.Queue] = [] def add_consumer(self): - """ - Adds a new consumer and returns its associated queue. + """Add a new consumer and return its associated queue. Returns: asyncio.Queue: The queue for the newly added consumer. @@ -49,15 +69,15 @@ class ProducerProcessor(FrameProcessor): return queue async def process_frame(self, frame: Frame, direction: FrameDirection): - """ - Processes an incoming frame and determines whether to produce it as a ProducerItem. + """Process an incoming frame and determine whether to produce it. - If the frame meets the produce criteria, it will be added to the consumer queues. - If passthrough is enabled, the frame will also be sent to consumers. + If the frame meets the filter criteria, it will be transformed and added + to all consumer queues. If passthrough is enabled, the original frame + will also be sent downstream. Args: - frame (Frame): The frame to process. - direction (FrameDirection): The direction of the frame. + frame: The frame to process. + direction: The direction of the frame flow. """ await super().process_frame(frame, direction) @@ -69,6 +89,7 @@ class ProducerProcessor(FrameProcessor): await self.push_frame(frame, direction) async def _produce(self, frame: Frame): + """Produce a frame to all consumers.""" for consumer in self._consumers: new_frame = await self._transformer(frame) await consumer.put(new_frame) diff --git a/src/pipecat/processors/text_transformer.py b/src/pipecat/processors/text_transformer.py index 9b563e187..7dcef31df 100644 --- a/src/pipecat/processors/text_transformer.py +++ b/src/pipecat/processors/text_transformer.py @@ -4,14 +4,20 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import Coroutine +"""Stateless text transformation processor for Pipecat.""" + +from typing import Callable, Coroutine, Union from pipecat.frames.frames import Frame, TextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class StatelessTextTransformer(FrameProcessor): - """This processor calls the given function on any text in a text frame. + """Processor that applies transformation functions to text frames. + + This processor intercepts TextFrame objects and applies a user-provided + transformation function to the text content. The function can be either + synchronous or asynchronous (coroutine). >>> async def print_frames(aggregator, frame): ... async for frame in aggregator.process_frame(frame): @@ -22,11 +28,25 @@ class StatelessTextTransformer(FrameProcessor): HELLO """ - def __init__(self, transform_fn): + def __init__( + self, transform_fn: Union[Callable[[str], str], Callable[[str], Coroutine[None, None, str]]] + ): + """Initialize the text transformer. + + Args: + transform_fn: Function to apply to text content. Can be synchronous + (str -> str) or asynchronous (str -> Coroutine[None, None, str]). + """ super().__init__() self._transform_fn = transform_fn async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames, applying transformation to text frames. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, TextFrame): diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 97ccd0cd4..856311392 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Transcript processing utilities for conversation recording and analysis. + +This module provides processors that convert speech and text frames into structured +transcript messages with timestamps, enabling conversation history tracking and analysis. +""" + from typing import List, Optional from loguru import logger @@ -30,7 +36,11 @@ class BaseTranscriptProcessor(FrameProcessor): """ def __init__(self, **kwargs): - """Initialize processor with empty message store.""" + """Initialize processor with empty message store. + + Args: + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._processed_messages: List[TranscriptionMessage] = [] self._register_event_handler("on_transcript_update") @@ -39,7 +49,7 @@ class BaseTranscriptProcessor(FrameProcessor): """Emit transcript updates for new messages. Args: - messages: New messages to emit in update + messages: New messages to emit in update. """ if messages: self._processed_messages.extend(messages) @@ -55,8 +65,8 @@ class UserTranscriptProcessor(BaseTranscriptProcessor): """Process TranscriptionFrames into user conversation messages. Args: - frame: Input frame to process - direction: Frame processing direction + frame: Input frame to process. + direction: Frame processing direction. """ await super().process_frame(frame, direction) @@ -77,14 +87,14 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): - The bot stops speaking (BotStoppedSpeakingFrame) - The bot is interrupted (StartInterruptionFrame) - The pipeline ends (EndFrame) - - Attributes: - _current_text_parts: List of text fragments being aggregated for current utterance - _aggregation_start_time: Timestamp when the current utterance began """ def __init__(self, **kwargs): - """Initialize processor with aggregation state.""" + """Initialize processor with aggregation state. + + Args: + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._current_text_parts: List[str] = [] self._aggregation_start_time: Optional[str] = None @@ -176,8 +186,8 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): - CancelFrame: Completes current utterance due to cancellation Args: - frame: Input frame to process - direction: Frame processing direction + frame: Input frame to process. + direction: Frame processing direction. """ await super().process_frame(frame, direction) @@ -245,7 +255,10 @@ class TranscriptProcessor: """Get the user transcript processor. Args: - **kwargs: Arguments specific to UserTranscriptProcessor + **kwargs: Arguments specific to UserTranscriptProcessor. + + Returns: + The user transcript processor instance. """ if self._user_processor is None: self._user_processor = UserTranscriptProcessor(**kwargs) @@ -262,7 +275,10 @@ class TranscriptProcessor: """Get the assistant transcript processor. Args: - **kwargs: Arguments specific to AssistantTranscriptProcessor + **kwargs: Arguments specific to AssistantTranscriptProcessor. + + Returns: + The assistant transcript processor instance. """ if self._assistant_processor is None: self._assistant_processor = AssistantTranscriptProcessor(**kwargs) @@ -279,10 +295,10 @@ class TranscriptProcessor: """Register event handler for both processors. Args: - event_name: Name of event to handle + event_name: Name of event to handle. Returns: - Decorator function that registers handler with both processors + Decorator function that registers handler with both processors. """ def decorator(handler): diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index e7b08f4a3..c692642cc 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""User idle detection and timeout handling for Pipecat.""" + import asyncio import inspect from typing import Awaitable, Callable, Union @@ -22,19 +24,12 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class UserIdleProcessor(FrameProcessor): """Monitors user inactivity and triggers callbacks after timeout periods. - Starts monitoring only after the first conversation activity (UserStartedSpeaking - or BotSpeaking). - - Args: - callback: Function to call when user is idle. Can be either: - - Basic callback(processor) -> None - - Retry callback(processor, retry_count) -> bool - Return True to continue monitoring for idle events, - Return False to stop the idle monitoring task - timeout: Seconds to wait before considering user idle - **kwargs: Additional arguments passed to FrameProcessor + This processor tracks user activity and triggers configurable callbacks when + users become idle. It starts monitoring only after the first conversation + activity and supports both basic and retry-based callback patterns. Example: + ``` # Retry callback: async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool: if retry_count < 3: @@ -50,6 +45,7 @@ class UserIdleProcessor(FrameProcessor): callback=handle_idle, timeout=5.0 ) + ``` """ def __init__( @@ -62,6 +58,17 @@ class UserIdleProcessor(FrameProcessor): timeout: float, **kwargs, ): + """Initialize the user idle processor. + + Args: + callback: Function to call when user is idle. Can be either: + - Basic callback(processor) -> None + - Retry callback(processor, retry_count) -> bool + Return True to continue monitoring for idle events, + Return False to stop the idle monitoring task + timeout: Seconds to wait before considering user idle. + **kwargs: Additional arguments passed to FrameProcessor. + """ super().__init__(**kwargs) self._callback = self._wrap_callback(callback) self._timeout = timeout @@ -107,7 +114,11 @@ class UserIdleProcessor(FrameProcessor): @property def retry_count(self) -> int: - """Returns the current retry count.""" + """Get the current retry count. + + Returns: + The number of times the idle callback has been triggered. + """ return self._retry_count async def _stop(self) -> None: @@ -120,8 +131,8 @@ class UserIdleProcessor(FrameProcessor): """Processes incoming frames and manages idle monitoring state. Args: - frame: The frame to process - direction: Direction of the frame flow + frame: The frame to process. + direction: Direction of the frame flow. """ await super().process_frame(frame, direction) diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index 175673e43..eebdbfb0b 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -32,12 +32,14 @@ class AIService(FrameProcessor): settings handling, session properties, and frame processing lifecycle. Subclasses should implement specific AI functionality while leveraging this base infrastructure. - - Args: - **kwargs: Additional arguments passed to the parent FrameProcessor. """ def __init__(self, **kwargs): + """Initialize the AI service. + + Args: + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) self._model_name: str = "" self._settings: Dict[str, Any] = {} diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 9d48d3c03..33b9c9e30 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -101,13 +101,6 @@ class AnthropicLLMService(LLMService): Provides inference capabilities with Claude models including support for function calling, vision processing, streaming responses, and prompt caching. Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex. - - Args: - api_key: Anthropic API key for authentication. - model: Model name to use. Defaults to "claude-sonnet-4-20250514". - params: Optional model parameters for inference. - client: Optional custom Anthropic client instance. - **kwargs: Additional arguments passed to parent LLMService. """ # Overriding the default adapter to use the Anthropic one. @@ -141,6 +134,15 @@ class AnthropicLLMService(LLMService): client=None, **kwargs, ): + """Initialize the Anthropic LLM service. + + Args: + api_key: Anthropic API key for authentication. + model: Model name to use. Defaults to "claude-sonnet-4-20250514". + params: Optional model parameters for inference. + client: Optional custom Anthropic client instance. + **kwargs: Additional arguments passed to parent LLMService. + """ super().__init__(**kwargs) params = params or AnthropicLLMService.InputParams() self._client = client or AsyncAnthropic( @@ -425,12 +427,6 @@ class AnthropicLLMContext(OpenAILLMContext): Extends OpenAILLMContext to handle Anthropic-specific features like system messages, prompt caching, and message format conversions. Manages conversation state and message history formatting. - - Args: - messages: Initial list of conversation messages. - tools: Available function calling tools. - tool_choice: Tool selection preference. - system: System message content. """ def __init__( @@ -441,6 +437,14 @@ class AnthropicLLMContext(OpenAILLMContext): *, system: Union[str, NotGiven] = NOT_GIVEN, ): + """Initialize the Anthropic LLM context. + + Args: + messages: Initial list of conversation messages. + tools: Available function calling tools. + tool_choice: Tool selection preference. + system: System message content. + """ super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) # For beta prompt caching. This is a counter that tracks the number of turns diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index 58b69fdf5..b34ec554d 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -1,10 +1,30 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AssemblyAI WebSocket API message models and connection parameters. + +This module defines Pydantic models for handling AssemblyAI's real-time +transcription WebSocket messages and connection configuration. +""" + from typing import List, Literal, Optional from pydantic import BaseModel, Field class Word(BaseModel): - """Represents a single word in a transcription with timing and confidence.""" + """Represents a single word in a transcription with timing and confidence. + + Parameters: + start: Start time of the word in milliseconds. + end: End time of the word in milliseconds. + text: The transcribed word text. + confidence: Confidence score for the word (0.0 to 1.0). + word_is_final: Whether this word is finalized and won't change. + """ start: int end: int @@ -14,13 +34,23 @@ class Word(BaseModel): class BaseMessage(BaseModel): - """Base class for all AssemblyAI WebSocket messages.""" + """Base class for all AssemblyAI WebSocket messages. + + Parameters: + type: The message type identifier. + """ type: str class BeginMessage(BaseMessage): - """Message sent when a new session begins.""" + """Message sent when a new session begins. + + Parameters: + type: Always "Begin" for this message type. + id: Unique session identifier. + expires_at: Unix timestamp when the session expires. + """ type: Literal["Begin"] = "Begin" id: str @@ -28,7 +58,17 @@ class BeginMessage(BaseMessage): class TurnMessage(BaseMessage): - """Message containing transcription data for a turn of speech.""" + """Message containing transcription data for a turn of speech. + + Parameters: + type: Always "Turn" for this message type. + turn_order: Sequential number of this turn in the session. + turn_is_formatted: Whether the transcript has been formatted. + end_of_turn: Whether this marks the end of a speaking turn. + transcript: The transcribed text for this turn. + end_of_turn_confidence: Confidence score for end-of-turn detection. + words: List of individual words with timing and confidence data. + """ type: Literal["Turn"] = "Turn" turn_order: int @@ -40,7 +80,13 @@ class TurnMessage(BaseMessage): class TerminationMessage(BaseMessage): - """Message sent when the session is terminated.""" + """Message sent when the session is terminated. + + Parameters: + type: Always "Termination" for this message type. + audio_duration_seconds: Total duration of audio processed. + session_duration_seconds: Total duration of the session. + """ type: Literal["Termination"] = "Termination" audio_duration_seconds: float @@ -52,6 +98,18 @@ AnyMessage = BeginMessage | TurnMessage | TerminationMessage class AssemblyAIConnectionParams(BaseModel): + """Configuration parameters for AssemblyAI WebSocket connection. + + Parameters: + sample_rate: Audio sample rate in Hz. Defaults to 16000. + encoding: Audio encoding format. Defaults to "pcm_s16le". + formatted_finals: Whether to enable transcript formatting. Defaults to True. + word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds. + end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection. + min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn. + max_turn_silence: Maximum silence duration before forcing end-of-turn. + """ + sample_rate: int = 16000 encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le" formatted_finals: bool = True diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 452d4cfb6..0e7103885 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AssemblyAI speech-to-text service implementation. + +This module provides integration with AssemblyAI's real-time speech-to-text +WebSocket API for streaming audio transcription. +""" + import asyncio import json from typing import Any, AsyncGenerator, Dict @@ -45,6 +51,13 @@ except ModuleNotFoundError as e: class AssemblyAISTTService(STTService): + """AssemblyAI real-time speech-to-text service. + + Provides real-time speech transcription using AssemblyAI's WebSocket API. + Supports both interim and final transcriptions with configurable parameters + for audio processing and connection management. + """ + def __init__( self, *, @@ -55,6 +68,16 @@ class AssemblyAISTTService(STTService): vad_force_turn_endpoint: bool = True, **kwargs, ): + """Initialize the AssemblyAI STT service. + + Args: + api_key: AssemblyAI API key for authentication. + language: Language code for transcription. Defaults to English (Language.EN). + api_endpoint_base_url: WebSocket endpoint URL. Defaults to AssemblyAI's streaming endpoint. + connection_params: Connection configuration parameters. Defaults to AssemblyAIConnectionParams(). + vad_force_turn_endpoint: Whether to force turn endpoint on VAD stop. Defaults to True. + **kwargs: Additional arguments passed to parent STTService class. + """ self._api_key = api_key self._language = language self._api_endpoint_base_url = api_endpoint_base_url @@ -75,22 +98,50 @@ class AssemblyAISTTService(STTService): self._chunk_size_bytes = 0 def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True if metrics generation is supported. + """ return True async def start(self, frame: StartFrame): + """Start the speech-to-text service. + + Args: + frame: Start frame to begin processing. + """ await super().start(frame) self._chunk_size_bytes = int(self._chunk_size_ms * self._sample_rate * 2 / 1000) await self._connect() async def stop(self, frame: EndFrame): + """Stop the speech-to-text service. + + Args: + frame: End frame to stop processing. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the speech-to-text service. + + Args: + frame: Cancel frame to abort processing. + """ await super().cancel(frame) await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text conversion. + + Args: + audio: Raw audio bytes to process. + + Yields: + None (processing handled via WebSocket messages). + """ self._audio_buffer.extend(audio) while len(self._audio_buffer) >= self._chunk_size_bytes: @@ -101,6 +152,12 @@ class AssemblyAISTTService(STTService): yield None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for VAD and metrics handling. + + Args: + frame: Frame to process. + direction: Direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): await self.start_ttfb_metrics() diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index 249fb81da..283cea601 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -104,12 +104,6 @@ class AWSBedrockLLMContext(OpenAILLMContext): Extends OpenAI LLM context to handle AWS Bedrock's specific message format and system message handling. Manages conversion between OpenAI and Bedrock message formats. - - Args: - messages: List of conversation messages in OpenAI format. - tools: List of available function calling tools. - tool_choice: Tool selection strategy or specific tool choice. - system: System message content for AWS Bedrock. """ def __init__( @@ -120,6 +114,14 @@ class AWSBedrockLLMContext(OpenAILLMContext): *, system: Optional[str] = None, ): + """Initialize AWS Bedrock LLM context. + + Args: + messages: List of conversation messages in OpenAI format. + tools: List of available function calling tools. + tool_choice: Tool selection strategy or specific tool choice. + system: System message content for AWS Bedrock. + """ super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) self.system = system @@ -656,16 +658,6 @@ class AWSBedrockLLMService(LLMService): Provides inference capabilities for AWS Bedrock models including Amazon Nova and Anthropic Claude. Supports streaming responses, function calling, and vision capabilities. - - Args: - model: The AWS Bedrock model identifier to use. - aws_access_key: AWS access key ID. If None, uses default credentials. - aws_secret_key: AWS secret access key. If None, uses default credentials. - aws_session_token: AWS session token for temporary credentials. - aws_region: AWS region for the Bedrock service. - params: Model parameters and configuration. - client_config: Custom boto3 client configuration. - **kwargs: Additional arguments passed to parent LLMService. """ # Overriding the default adapter to use the Anthropic one. @@ -702,6 +694,18 @@ class AWSBedrockLLMService(LLMService): client_config: Optional[Config] = None, **kwargs, ): + """Initialize the AWS Bedrock LLM service. + + Args: + model: The AWS Bedrock model identifier to use. + aws_access_key: AWS access key ID. If None, uses default credentials. + aws_secret_key: AWS secret access key. If None, uses default credentials. + aws_session_token: AWS session token for temporary credentials. + aws_region: AWS region for the Bedrock service. + params: Model parameters and configuration. + client_config: Custom boto3 client configuration. + **kwargs: Additional arguments passed to parent LLMService. + """ super().__init__(**kwargs) params = params or AWSBedrockLLMService.InputParams() diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index c4170ebad..a7f8fea97 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Transcribe Speech-to-Text service implementation. + +This module provides a WebSocket-based connection to AWS Transcribe for real-time +speech-to-text transcription with support for multiple languages and audio formats. +""" + import asyncio import json import os @@ -37,6 +43,13 @@ except ModuleNotFoundError as e: class AWSTranscribeSTTService(STTService): + """AWS Transcribe Speech-to-Text service using WebSocket streaming. + + Provides real-time speech transcription using AWS Transcribe's streaming API. + Supports multiple languages, configurable sample rates, and both interim and + final transcription results. + """ + def __init__( self, *, @@ -48,6 +61,17 @@ class AWSTranscribeSTTService(STTService): language: Language = Language.EN, **kwargs, ): + """Initialize the AWS Transcribe STT service. + + Args: + api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable. + aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable. + aws_session_token: AWS session token for temporary credentials. If None, uses AWS_SESSION_TOKEN environment variable. + region: AWS region for the service. Defaults to "us-east-1". + sample_rate: Audio sample rate in Hz. Must be 8000 or 16000. Defaults to 16000. + language: Language for transcription. Defaults to English. + **kwargs: Additional arguments passed to parent STTService class. + """ super().__init__(**kwargs) self._settings = { @@ -79,14 +103,28 @@ class AWSTranscribeSTTService(STTService): self._receive_task = None def get_service_encoding(self, encoding: str) -> str: - """Convert internal encoding format to AWS Transcribe format.""" + """Convert internal encoding format to AWS Transcribe format. + + Args: + encoding: Internal encoding format string. + + Returns: + AWS Transcribe compatible encoding format. + """ encoding_map = { "linear16": "pcm", # AWS expects "pcm" for 16-bit linear PCM } return encoding_map.get(encoding, encoding) async def start(self, frame: StartFrame): - """Initialize the connection when the service starts.""" + """Initialize the connection when the service starts. + + Args: + frame: Start frame signaling service initialization. + + Raises: + RuntimeError: If WebSocket connection cannot be established after retries. + """ await super().start(frame) logger.info("Starting AWS Transcribe service...") retry_count = 0 @@ -108,15 +146,32 @@ class AWSTranscribeSTTService(STTService): raise RuntimeError("Failed to establish WebSocket connection after multiple attempts") async def stop(self, frame: EndFrame): + """Stop the service and disconnect from AWS Transcribe. + + Args: + frame: End frame signaling service shutdown. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the service and disconnect from AWS Transcribe. + + Args: + frame: Cancel frame signaling service cancellation. + """ await super().cancel(frame) await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Process audio data and send to AWS Transcribe""" + """Process audio data and send to AWS Transcribe. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + ErrorFrame: If processing fails or connection issues occur. + """ try: # Ensure WebSocket is connected if not self._ws_client or not self._ws_client.open: @@ -255,7 +310,14 @@ class AWSTranscribeSTTService(STTService): self._ws_client = None def language_to_service_language(self, language: Language) -> str | None: - """Convert internal language enum to AWS Transcribe language code.""" + """Convert internal language enum to AWS Transcribe language code. + + Args: + language: Internal language enumeration value. + + Returns: + AWS Transcribe compatible language code, or None if unsupported. + """ language_map = { Language.EN: "en-US", Language.ES: "es-US", diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index 762e8b9e4..ce89dea9e 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Polly text-to-speech service implementation. + +This module provides integration with Amazon Polly for text-to-speech synthesis, +supporting multiple languages, voices, and SSML features. +""" + import asyncio import os from typing import AsyncGenerator, List, Optional @@ -33,6 +39,14 @@ except ModuleNotFoundError as e: def language_to_aws_language(language: Language) -> Optional[str]: + """Convert a Language enum to AWS Polly language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding AWS Polly language code, or None if not supported. + """ language_map = { # Arabic Language.AR: "arb", @@ -109,7 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]: class AWSPollyTTSService(TTSService): + """AWS Polly text-to-speech service. + + Provides text-to-speech synthesis using Amazon Polly with support for + multiple languages, voices, SSML features, and voice customization + options including prosody controls. + """ + class InputParams(BaseModel): + """Input parameters for AWS Polly TTS configuration. + + Parameters: + engine: TTS engine to use ('standard', 'neural', etc.). + language: Language for synthesis. Defaults to English. + pitch: Voice pitch adjustment (for standard engine only). + rate: Speech rate adjustment. + volume: Voice volume adjustment. + lexicon_names: List of pronunciation lexicons to apply. + """ + engine: Optional[str] = None language: Optional[Language] = Language.EN pitch: Optional[str] = None @@ -129,6 +161,18 @@ class AWSPollyTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initializes the AWS Polly TTS service. + + Args: + api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable. + aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable. + aws_session_token: AWS session token for temporary credentials. + region: AWS region for Polly service. Defaults to 'us-east-1'. + voice_id: Voice ID to use for synthesis. Defaults to 'Joanna'. + sample_rate: Audio sample rate. If None, uses service default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to parent TTSService class. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or AWSPollyTTSService.InputParams() @@ -174,9 +218,22 @@ class AWSPollyTTSService(TTSService): ) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as AWS Polly service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to AWS Polly language format. + + Args: + language: The language to convert. + + Returns: + The AWS Polly-specific language code, or None if not supported. + """ return language_to_aws_language(language) def _construct_ssml(self, text: str) -> str: @@ -214,6 +271,15 @@ class AWSPollyTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using AWS Polly. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + def read_audio_data(**args): response = self._polly_client.synthesize_speech(**args) if "AudioStream" in response: @@ -277,7 +343,14 @@ class AWSPollyTTSService(TTSService): class PollyTTSService(AWSPollyTTSService): + """Deprecated alias for AWSPollyTTSService.""" + def __init__(self, **kwargs): + """Initialize the deprecated PollyTTSService. + + Args: + **kwargs: All arguments passed to AWSPollyTTSService. + """ super().__init__(**kwargs) import warnings diff --git a/src/pipecat/services/aws/utils.py b/src/pipecat/services/aws/utils.py index db69456e9..cfd36b417 100644 --- a/src/pipecat/services/aws/utils.py +++ b/src/pipecat/services/aws/utils.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Transcribe utility functions and classes for WebSocket streaming. + +This module provides utilities for creating presigned URLs, building event messages, +and handling AWS event stream protocol for real-time transcription services. +""" + import binascii import datetime import hashlib @@ -29,7 +35,31 @@ def get_presigned_url( show_speaker_label: bool = False, enable_channel_identification: bool = False, ) -> str: - """Create a presigned URL for AWS Transcribe streaming.""" + """Create a presigned URL for AWS Transcribe streaming. + + Args: + region: AWS region for the service. + credentials: Dictionary containing AWS credentials with keys: + - access_key: AWS access key ID + - secret_key: AWS secret access key + - session_token: AWS session token (optional) + language_code: Language code for transcription (e.g., "en-US"). + media_encoding: Audio encoding format. Defaults to "pcm". + sample_rate: Audio sample rate in Hz. Defaults to 16000. + number_of_channels: Number of audio channels. Defaults to 1. + enable_partial_results_stabilization: Whether to enable partial result stabilization. + partial_results_stability: Stability level for partial results. + vocabulary_name: Custom vocabulary name to use. + vocabulary_filter_name: Vocabulary filter name to apply. + show_speaker_label: Whether to include speaker labels. + enable_channel_identification: Whether to enable channel identification. + + Returns: + Presigned WebSocket URL for AWS Transcribe streaming. + + Raises: + ValueError: If required AWS credentials are missing. + """ access_key = credentials.get("access_key") secret_key = credentials.get("secret_key") session_token = credentials.get("session_token") @@ -58,9 +88,23 @@ def get_presigned_url( class AWSTranscribePresignedURL: + """Generator for AWS Transcribe presigned WebSocket URLs. + + Handles AWS Signature Version 4 signing process to create authenticated + WebSocket URLs for streaming transcription requests. + """ + def __init__( self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1" ): + """Initialize the presigned URL generator. + + Args: + access_key: AWS access key ID. + secret_key: AWS secret access key. + session_token: AWS session token for temporary credentials. + region: AWS region for the service. Defaults to "us-east-1". + """ self.access_key = access_key self.secret_key = secret_key self.session_token = session_token @@ -96,6 +140,23 @@ class AWSTranscribePresignedURL: enable_partial_results_stabilization: bool = False, partial_results_stability: str = "", ) -> str: + """Generate a presigned WebSocket URL for AWS Transcribe. + + Args: + sample_rate: Audio sample rate in Hz. + language_code: Language code for transcription. + media_encoding: Audio encoding format. + vocabulary_name: Custom vocabulary name. + vocabulary_filter_name: Vocabulary filter name. + show_speaker_label: Whether to include speaker labels. + enable_channel_identification: Whether to enable channel identification. + number_of_channels: Number of audio channels. + enable_partial_results_stabilization: Whether to enable partial result stabilization. + partial_results_stability: Stability level for partial results. + + Returns: + Presigned WebSocket URL with authentication parameters. + """ self.endpoint = f"wss://transcribestreaming.{self.region}.amazonaws.com:8443" self.host = f"transcribestreaming.{self.region}.amazonaws.com:8443" @@ -172,7 +233,15 @@ class AWSTranscribePresignedURL: def get_headers(header_name: str, header_value: str) -> bytearray: - """Build a header following AWS event stream format.""" + """Build a header following AWS event stream format. + + Args: + header_name: Name of the header. + header_value: Value of the header. + + Returns: + Encoded header as a bytearray following AWS event stream protocol. + """ name = header_name.encode("utf-8") name_byte_length = bytes([len(name)]) value_type = bytes([7]) # 7 represents a string @@ -190,9 +259,21 @@ def get_headers(header_name: str, header_value: str) -> bytearray: def build_event_message(payload: bytes) -> bytes: - """ - Build an event message for AWS Transcribe streaming. - Matches AWS sample: https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py + """Build an event message for AWS Transcribe streaming. + + Creates a properly formatted AWS event stream message containing audio data + for real-time transcription. Follows the AWS event stream protocol with + prelude, headers, payload, and CRC checksums. + + Args: + payload: Raw audio bytes to include in the event message. + + Returns: + Complete event message as bytes, ready to send via WebSocket. + + Note: + Implementation matches AWS sample: + https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py """ # Build headers content_type_header = get_headers(":content-type", "application/octet-stream") @@ -235,6 +316,22 @@ def build_event_message(payload: bytes) -> bytes: def decode_event(message): + """Decode an AWS event stream message. + + Parses an AWS event stream message to extract headers and payload, + verifying CRC checksums for data integrity. + + Args: + message: Raw event stream message bytes received from AWS. + + Returns: + Tuple containing: + - Dictionary of parsed headers + - Dictionary of parsed JSON payload + + Raises: + AssertionError: If CRC checksum verification fails. + """ # Extract the prelude, headers, payload and CRC prelude = message[:8] total_length, headers_length = struct.unpack(">II", prelude) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index c28d218c8..77e9575b5 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -96,7 +96,13 @@ class AWSNovaSonicUnhandledFunctionException(Exception): class ContentType(Enum): - """Content types supported by AWS Nova Sonic.""" + """Content types supported by AWS Nova Sonic. + + Parameters: + AUDIO: Audio content type. + TEXT: Text content type. + TOOL: Tool content type. + """ AUDIO = "AUDIO" TEXT = "TEXT" @@ -104,7 +110,12 @@ class ContentType(Enum): class TextStage(Enum): - """Text generation stages in AWS Nova Sonic responses.""" + """Text generation stages in AWS Nova Sonic responses. + + Parameters: + FINAL: Final text that has been fully generated. + SPECULATIVE: Speculative text that is still being generated. + """ FINAL = "FINAL" # what has been said SPECULATIVE = "SPECULATIVE" # what's planned to be said @@ -127,6 +138,7 @@ class CurrentContent: text_content: str # starts as None, then fills in if text def __str__(self): + """String representation of the current content.""" return ( f"CurrentContent(\n" f" type={self.type.name},\n" @@ -172,18 +184,6 @@ class AWSNovaSonicLLMService(LLMService): Provides bidirectional audio streaming, real-time transcription, text generation, and function calling capabilities using AWS Nova Sonic model. - - Args: - secret_access_key: AWS secret access key for authentication. - access_key_id: AWS access key ID for authentication. - region: AWS region where the service is hosted. - model: Model identifier. Defaults to "amazon.nova-sonic-v1:0". - voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy. - params: Model parameters for audio configuration and inference. - system_instruction: System-level instruction for the model. - tools: Available tools/functions for the model to use. - send_transcription_frames: Whether to emit transcription frames. - **kwargs: Additional arguments passed to the parent LLMService. """ # Override the default adapter to use the AWSNovaSonicLLMAdapter one @@ -203,6 +203,20 @@ class AWSNovaSonicLLMService(LLMService): send_transcription_frames: bool = True, **kwargs, ): + """Initializes the AWS Nova Sonic LLM service. + + Args: + secret_access_key: AWS secret access key for authentication. + access_key_id: AWS access key ID for authentication. + region: AWS region where the service is hosted. + model: Model identifier. Defaults to "amazon.nova-sonic-v1:0". + voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy. + params: Model parameters for audio configuration and inference. + system_instruction: System-level instruction for the model. + tools: Available tools/functions for the model to use. + send_transcription_frames: Whether to emit transcription frames. + **kwargs: Additional arguments passed to the parent LLMService. + """ super().__init__(**kwargs) self._secret_access_key = secret_access_key self._access_key_id = access_key_id diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 327da4e40..e23a18362 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -41,7 +41,14 @@ from pipecat.services.openai.llm import ( class Role(Enum): - """Roles supported in AWS Nova Sonic conversations.""" + """Roles supported in AWS Nova Sonic conversations. + + Parameters: + SYSTEM: System-level messages (not used in conversation history). + USER: Messages sent by the user. + ASSISTANT: Messages sent by the assistant. + TOOL: Messages sent by tools (not used in conversation history). + """ SYSTEM = "SYSTEM" USER = "USER" @@ -80,14 +87,16 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): Extends OpenAI context with Nova Sonic-specific message handling, conversation history management, and text buffering capabilities. - - Args: - messages: Initial messages for the context. - tools: Available tools for the context. - **kwargs: Additional arguments passed to parent class. """ def __init__(self, messages=None, tools=None, **kwargs): + """Initialize AWS Nova Sonic LLM context. + + Args: + messages: Initial messages for the context. + tools: Available tools for the context. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(messages=messages, tools=tools, **kwargs) self.__setup_local() diff --git a/src/pipecat/services/azure/common.py b/src/pipecat/services/azure/common.py index 054463257..a6f1eeedd 100644 --- a/src/pipecat/services/azure/common.py +++ b/src/pipecat/services/azure/common.py @@ -4,14 +4,22 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import Optional +"""Language conversion utilities for Azure services.""" -from loguru import logger +from typing import Optional from pipecat.transcriptions.language import Language def language_to_azure_language(language: Language) -> Optional[str]: + """Convert a Language enum to Azure language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Azure language code, or None if not supported. + """ language_map = { # Afrikaans Language.AF: "af-ZA", diff --git a/src/pipecat/services/azure/image.py b/src/pipecat/services/azure/image.py index a1bae3af6..f07b86f83 100644 --- a/src/pipecat/services/azure/image.py +++ b/src/pipecat/services/azure/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure OpenAI image generation service implementation. + +This module provides integration with Azure's OpenAI image generation API +using REST endpoints for creating images from text prompts. +""" + import asyncio import io from typing import AsyncGenerator @@ -17,6 +23,13 @@ from pipecat.services.image_service import ImageGenService class AzureImageGenServiceREST(ImageGenService): + """Azure OpenAI REST-based image generation service. + + Provides image generation using Azure's OpenAI service via REST API. + Supports asynchronous image generation with polling for completion + and automatic image download and processing. + """ + def __init__( self, *, @@ -27,6 +40,16 @@ class AzureImageGenServiceREST(ImageGenService): aiohttp_session: aiohttp.ClientSession, api_version="2023-06-01-preview", ): + """Initialize the AzureImageGenServiceREST. + + Args: + image_size: Size specification for generated images (e.g., "1024x1024"). + api_key: Azure OpenAI API key for authentication. + endpoint: Azure OpenAI endpoint URL. + model: The image generation model to use. + aiohttp_session: Shared aiohttp session for HTTP requests. + api_version: Azure API version string. Defaults to "2023-06-01-preview". + """ super().__init__() self._api_key = api_key @@ -37,6 +60,15 @@ class AzureImageGenServiceREST(ImageGenService): self._aiohttp_session = aiohttp_session async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate an image from a text prompt using Azure OpenAI. + + Args: + prompt: The text prompt describing the desired image. + + Yields: + URLImageRawFrame containing the generated image data, or + ErrorFrame if generation fails. + """ url = f"{self._azure_endpoint}openai/images/generations:submit?api-version={self._api_version}" headers = {"api-key": self._api_key, "Content-Type": "application/json"} diff --git a/src/pipecat/services/azure/llm.py b/src/pipecat/services/azure/llm.py index bc1242044..a4b93f2a4 100644 --- a/src/pipecat/services/azure/llm.py +++ b/src/pipecat/services/azure/llm.py @@ -17,13 +17,6 @@ class AzureLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Azure's OpenAI endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing Azure OpenAI. - endpoint: The Azure endpoint URL. - model: The model identifier to use. - api_version: Azure API version. Defaults to "2024-09-01-preview". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -35,6 +28,15 @@ class AzureLLMService(OpenAILLMService): api_version: str = "2024-09-01-preview", **kwargs, ): + """Initialize the Azure LLM service. + + Args: + api_key: The API key for accessing Azure OpenAI. + endpoint: The Azure endpoint URL. + model: The model identifier to use. + api_version: Azure API version. Defaults to "2024-09-01-preview". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ # Initialize variables before calling parent __init__() because that # will call create_client() and we need those values there. self._endpoint = endpoint diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index abd8acbd3..415f91550 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure Speech-to-Text service implementation for Pipecat. + +This module provides speech-to-text functionality using Azure Cognitive Services +Speech SDK for real-time audio transcription. +""" + import asyncio from typing import AsyncGenerator, Optional @@ -40,6 +46,13 @@ except ModuleNotFoundError as e: class AzureSTTService(STTService): + """Azure Speech-to-Text service for real-time audio transcription. + + This service uses Azure Cognitive Services Speech SDK to convert speech + audio into text transcriptions. It supports continuous recognition and + provides real-time transcription results with timing information. + """ + def __init__( self, *, @@ -49,6 +62,15 @@ class AzureSTTService(STTService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the Azure STT service. + + Args: + api_key: Azure Cognitive Services subscription key. + region: Azure region for the Speech service (e.g., 'eastus'). + language: Language for speech recognition. Defaults to English (US). + sample_rate: Audio sample rate in Hz. If None, uses service default. + **kwargs: Additional arguments passed to parent STTService. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._speech_config = SpeechConfig( @@ -66,9 +88,25 @@ class AzureSTTService(STTService): } def can_generate_metrics(self) -> bool: + """Check if this service can generate performance metrics. + + Returns: + True as this service supports metrics generation. + """ return True async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text conversion. + + Feeds audio data to the Azure speech recognizer for processing. + Recognition results are handled asynchronously through callbacks. + + Args: + audio: Raw audio bytes to process. + + Yields: + None - actual transcription frames are pushed via callbacks. + """ await self.start_processing_metrics() await self.start_ttfb_metrics() if self._audio_stream: @@ -76,6 +114,14 @@ class AzureSTTService(STTService): yield None async def start(self, frame: StartFrame): + """Start the speech recognition service. + + Initializes the Azure speech recognizer with audio stream configuration + and begins continuous speech recognition. + + Args: + frame: Frame indicating the start of processing. + """ await super().start(frame) if self._audio_stream: @@ -93,6 +139,13 @@ class AzureSTTService(STTService): self._speech_recognizer.start_continuous_recognition_async() async def stop(self, frame: EndFrame): + """Stop the speech recognition service. + + Cleanly shuts down the Azure speech recognizer and closes audio streams. + + Args: + frame: Frame indicating the end of processing. + """ await super().stop(frame) if self._speech_recognizer: @@ -102,6 +155,13 @@ class AzureSTTService(STTService): self._audio_stream.close() async def cancel(self, frame: CancelFrame): + """Cancel the speech recognition service. + + Immediately stops recognition and closes resources. + + Args: + frame: Frame indicating cancellation. + """ await super().cancel(frame) if self._speech_recognizer: diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index e8be685a6..07e706a9b 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure Cognitive Services Text-to-Speech service implementations.""" + import asyncio from typing import AsyncGenerator, Optional @@ -39,6 +41,15 @@ except ModuleNotFoundError as e: def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputFormat: + """Convert sample rate to Azure speech synthesis output format. + + Args: + sample_rate: Sample rate in Hz. + + Returns: + Corresponding Azure SpeechSynthesisOutputFormat enum value. + Defaults to Raw24Khz16BitMonoPcm if sample rate not found. + """ sample_rate_map = { 8000: SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm, 16000: SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm, @@ -51,7 +62,26 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma class AzureBaseTTSService(TTSService): + """Base class for Azure Cognitive Services text-to-speech implementations. + + Provides common functionality for Azure TTS services including SSML + construction, voice configuration, and parameter management. + """ + class InputParams(BaseModel): + """Input parameters for Azure TTS voice configuration. + + Parameters: + emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). + language: Language for synthesis. Defaults to English (US). + pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high"). + rate: Speech rate multiplier. Defaults to "1.05". + role: Voice role for expression (e.g., "YoungAdultFemale"). + style: Speaking style (e.g., "cheerful", "sad", "excited"). + style_degree: Intensity of the speaking style (0.01 to 2.0). + volume: Volume level (e.g., "+20%", "loud", "x-soft"). + """ + emphasis: Optional[str] = None language: Optional[Language] = Language.EN_US pitch: Optional[str] = None @@ -71,6 +101,16 @@ class AzureBaseTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Azure TTS service with configuration parameters. + + Args: + api_key: Azure Cognitive Services subscription key. + region: Azure region identifier (e.g., "eastus", "westus2"). + voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural". + sample_rate: Audio sample rate in Hz. If None, uses service default. + params: Voice and synthesis parameters configuration. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or AzureBaseTTSService.InputParams() @@ -94,9 +134,22 @@ class AzureBaseTTSService(TTSService): self._speech_synthesizer = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Azure TTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Azure language format. + + Args: + language: The language to convert. + + Returns: + The Azure-specific language code, or None if not supported. + """ return language_to_azure_language(language) def _construct_ssml(self, text: str) -> str: @@ -146,13 +199,30 @@ class AzureBaseTTSService(TTSService): class AzureTTSService(AzureBaseTTSService): + """Azure Cognitive Services streaming TTS service. + + Provides real-time text-to-speech synthesis using Azure's WebSocket-based + streaming API. Audio chunks are streamed as they become available for + lower latency playback. + """ + def __init__(self, **kwargs): + """Initialize the Azure streaming TTS service. + + Args: + **kwargs: All arguments passed to AzureBaseTTSService parent class. + """ super().__init__(**kwargs) self._speech_config = None self._speech_synthesizer = None self._audio_queue = asyncio.Queue() async def start(self, frame: StartFrame): + """Start the Azure TTS service and initialize speech synthesizer. + + Args: + frame: Start frame containing initialization parameters. + """ await super().start(frame) if self._speech_config: @@ -183,24 +253,33 @@ class AzureTTSService(AzureBaseTTSService): self._speech_synthesizer.synthesis_canceled.connect(self._handle_canceled) def _handle_synthesizing(self, evt): - """Handle audio chunks as they arrive""" + """Handle audio chunks as they arriv.""" if evt.result and evt.result.audio_data: self._audio_queue.put_nowait(evt.result.audio_data) def _handle_completed(self, evt): - """Handle synthesis completion""" + """Handle synthesis completion.""" self._audio_queue.put_nowait(None) # Signal completion def _handle_canceled(self, evt): - """Handle synthesis cancellation""" + """Handle synthesis cancellation.""" logger.error(f"Speech synthesis canceled: {evt.result.cancellation_details.reason}") self._audio_queue.put_nowait(None) async def flush_audio(self): + """Flush any pending audio data.""" logger.trace(f"{self}: flushing audio") @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Azure's streaming synthesis. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing synthesized speech data. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -244,12 +323,29 @@ class AzureTTSService(AzureBaseTTSService): class AzureHttpTTSService(AzureBaseTTSService): + """Azure Cognitive Services HTTP-based TTS service. + + Provides text-to-speech synthesis using Azure's HTTP API for simpler, + non-streaming synthesis. Suitable for use cases where streaming is not + required and simpler integration is preferred. + """ + def __init__(self, **kwargs): + """Initialize the Azure HTTP TTS service. + + Args: + **kwargs: All arguments passed to AzureBaseTTSService parent class. + """ super().__init__(**kwargs) self._speech_config = None self._speech_synthesizer = None async def start(self, frame: StartFrame): + """Start the Azure HTTP TTS service and initialize speech synthesizer. + + Args: + frame: Start frame containing initialization parameters. + """ await super().start(frame) if self._speech_config: @@ -269,6 +365,14 @@ class AzureHttpTTSService(AzureBaseTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Azure's HTTP synthesis API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the complete synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") await self.start_ttfb_metrics() diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index 104e4b2c5..5ca9cce0f 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Cartesia Speech-to-Text service implementation. + +This module provides a WebSocket-based STT service that integrates with +the Cartesia Live transcription API for real-time speech recognition. +""" + import asyncio import json import urllib.parse @@ -30,6 +36,12 @@ from pipecat.utils.tracing.service_decorators import traced_stt class CartesiaLiveOptions: + """Configuration options for Cartesia Live STT service. + + Manages transcription parameters including model selection, language, + audio encoding format, and sample rate settings. + """ + def __init__( self, *, @@ -39,6 +51,15 @@ class CartesiaLiveOptions: sample_rate: int = 16000, **kwargs, ): + """Initialize CartesiaLiveOptions with default or provided parameters. + + Args: + model: The transcription model to use. Defaults to "ink-whisper". + language: Target language for transcription. Defaults to English. + encoding: Audio encoding format. Defaults to "pcm_s16le". + sample_rate: Audio sample rate in Hz. Defaults to 16000. + **kwargs: Additional parameters for the transcription service. + """ self.model = model self.language = language self.encoding = encoding @@ -46,6 +67,11 @@ class CartesiaLiveOptions: self.additional_params = kwargs def to_dict(self): + """Convert options to dictionary format. + + Returns: + Dictionary containing all configuration parameters. + """ params = { "model": self.model, "language": self.language if isinstance(self.language, str) else self.language.value, @@ -56,19 +82,48 @@ class CartesiaLiveOptions: return params def items(self): + """Get configuration items as key-value pairs. + + Returns: + Iterator of (key, value) tuples for all configuration parameters. + """ return self.to_dict().items() def get(self, key, default=None): + """Get a configuration value by key. + + Args: + key: The configuration parameter name to retrieve. + default: Default value if key is not found. + + Returns: + The configuration value or default if not found. + """ if hasattr(self, key): return getattr(self, key) return self.additional_params.get(key, default) @classmethod def from_json(cls, json_str: str) -> "CartesiaLiveOptions": + """Create options from JSON string. + + Args: + json_str: JSON string containing configuration parameters. + + Returns: + New CartesiaLiveOptions instance with parsed parameters. + """ return cls(**json.loads(json_str)) class CartesiaSTTService(STTService): + """Speech-to-text service using Cartesia Live API. + + Provides real-time speech transcription through WebSocket connection + to Cartesia's Live transcription service. Supports both interim and + final transcriptions with configurable models and languages. + """ + def __init__( self, *, @@ -78,6 +133,15 @@ class CartesiaSTTService(STTService): live_options: Optional[CartesiaLiveOptions] = None, **kwargs, ): + """Initialize CartesiaSTTService with API key and options. + + Args: + api_key: Authentication key for Cartesia API. + base_url: Custom API endpoint URL. If empty, uses default. + sample_rate: Audio sample rate in Hz. Defaults to 16000. + live_options: Configuration options for transcription service. + **kwargs: Additional arguments passed to parent STTService. + """ sample_rate = sample_rate or (live_options.sample_rate if live_options else None) super().__init__(sample_rate=sample_rate, **kwargs) @@ -108,21 +172,49 @@ class CartesiaSTTService(STTService): self._receiver_task = None def can_generate_metrics(self) -> bool: + """Check if the service can generate processing metrics. + + Returns: + True, indicating metrics are supported. + """ return True async def start(self, frame: StartFrame): + """Start the STT service and establish connection. + + Args: + frame: Frame indicating service should start. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the STT service and close connection. + + Args: + frame: Frame indicating service should stop. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the STT service and close connection. + + Args: + frame: Frame indicating service should be cancelled. + """ await super().cancel(frame) await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None - transcription results are handled via WebSocket responses. + """ # If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again if not self._connection or self._connection.closed: await self._connect() @@ -225,10 +317,17 @@ class CartesiaSTTService(STTService): self._connection = None async def start_metrics(self): + """Start performance metrics collection for transcription processing.""" await self.start_ttfb_metrics() await self.start_processing_metrics() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle speech events. + + Args: + frame: The frame to process. + direction: Direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 0edbb1f11..68cab0600 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -90,19 +90,6 @@ class CartesiaTTSService(AudioContextWordTTSService): Provides text-to-speech using Cartesia's streaming WebSocket API. Supports word-level timestamps, audio context management, and various voice customization options including speed and emotion controls. - - Args: - api_key: Cartesia API key for authentication. - voice_id: ID of the voice to use for synthesis. - cartesia_version: API version string for Cartesia service. - url: WebSocket URL for Cartesia TTS API. - model: TTS model to use (e.g., "sonic-2"). - sample_rate: Audio sample rate. If None, uses default. - encoding: Audio encoding format. - container: Audio container format. - params: Additional input parameters for voice customization. - text_aggregator: Custom text aggregator for processing input text. - **kwargs: Additional arguments passed to the parent service. """ class InputParams(BaseModel): @@ -133,6 +120,21 @@ class CartesiaTTSService(AudioContextWordTTSService): text_aggregator: Optional[BaseTextAggregator] = None, **kwargs, ): + """Initialize the Cartesia TTS service. + + Args: + api_key: Cartesia API key for authentication. + voice_id: ID of the voice to use for synthesis. + cartesia_version: API version string for Cartesia service. + url: WebSocket URL for Cartesia TTS API. + model: TTS model to use (e.g., "sonic-2"). + sample_rate: Audio sample rate. If None, uses default. + encoding: Audio encoding format. + container: Audio container format. + params: Additional input parameters for voice customization. + text_aggregator: Custom text aggregator for processing input text. + **kwargs: Additional arguments passed to the parent service. + """ # Aggregating sentences still gives cleaner-sounding results and fewer # artifacts than streaming one word at a time. On average, waiting for a # full sentence should only "cost" us 15ms or so with GPT-4o or a Llama @@ -404,18 +406,6 @@ class CartesiaHttpTTSService(TTSService): Provides text-to-speech using Cartesia's HTTP API for simpler, non-streaming synthesis. Suitable for use cases where streaming is not required and simpler integration is preferred. - - Args: - api_key: Cartesia API key for authentication. - voice_id: ID of the voice to use for synthesis. - model: TTS model to use (e.g., "sonic-2"). - base_url: Base URL for Cartesia HTTP API. - cartesia_version: API version string for Cartesia service. - sample_rate: Audio sample rate. If None, uses default. - encoding: Audio encoding format. - container: Audio container format. - params: Additional input parameters for voice customization. - **kwargs: Additional arguments passed to the parent TTSService. """ class InputParams(BaseModel): @@ -445,6 +435,20 @@ class CartesiaHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Cartesia HTTP TTS service. + + Args: + api_key: Cartesia API key for authentication. + voice_id: ID of the voice to use for synthesis. + model: TTS model to use (e.g., "sonic-2"). + base_url: Base URL for Cartesia HTTP API. + cartesia_version: API version string for Cartesia service. + sample_rate: Audio sample rate. If None, uses default. + encoding: Audio encoding format. + container: Audio container format. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or CartesiaHttpTTSService.InputParams() diff --git a/src/pipecat/services/cerebras/llm.py b/src/pipecat/services/cerebras/llm.py index fa3802891..fb8e21a2b 100644 --- a/src/pipecat/services/cerebras/llm.py +++ b/src/pipecat/services/cerebras/llm.py @@ -21,12 +21,6 @@ class CerebrasLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Cerebras's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing Cerebras's API. - base_url: The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1". - model: The model identifier to use. Defaults to "llama-3.3-70b". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -37,6 +31,14 @@ class CerebrasLLMService(OpenAILLMService): model: str = "llama-3.3-70b", **kwargs, ): + """Initialize the Cerebras LLM service. + + Args: + api_key: The API key for accessing Cerebras's API. + base_url: The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1". + model: The model identifier to use. Defaults to "llama-3.3-70b". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index da7ec535f..d897d8c92 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -48,15 +48,6 @@ class DeepgramSTTService(STTService): Provides real-time speech recognition using Deepgram's WebSocket API. Supports configurable models, languages, VAD events, and various audio processing options. - - Args: - api_key: Deepgram API key for authentication. - url: Deprecated. Use base_url instead. - base_url: Custom Deepgram API base URL. - sample_rate: Audio sample rate. If None, uses default or live_options value. - live_options: Deepgram LiveOptions for detailed configuration. - addons: Additional Deepgram features to enable. - **kwargs: Additional arguments passed to the parent STTService. """ def __init__( @@ -70,6 +61,17 @@ class DeepgramSTTService(STTService): addons: Optional[Dict] = None, **kwargs, ): + """Initialize the Deepgram STT service. + + Args: + api_key: Deepgram API key for authentication. + url: Deprecated. Use base_url instead. + base_url: Custom Deepgram API base URL. + sample_rate: Audio sample rate. If None, uses default or live_options value. + live_options: Deepgram LiveOptions for detailed configuration. + addons: Additional Deepgram features to enable. + **kwargs: Additional arguments passed to the parent STTService. + """ sample_rate = sample_rate or (live_options.sample_rate if live_options else None) super().__init__(sample_rate=sample_rate, **kwargs) diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index a684a340e..5819e4123 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Deepgram text-to-speech service implementation. + +This module provides integration with Deepgram's text-to-speech API +for generating speech from text using various voice models. +""" + from typing import AsyncGenerator, Optional from loguru import logger @@ -27,6 +33,13 @@ except ModuleNotFoundError as e: class DeepgramTTSService(TTSService): + """Deepgram text-to-speech service. + + Provides text-to-speech synthesis using Deepgram's streaming API. + Supports various voice models and audio encoding formats with + configurable sample rates and quality settings. + """ + def __init__( self, *, @@ -37,6 +50,16 @@ class DeepgramTTSService(TTSService): encoding: str = "linear16", **kwargs, ): + """Initialize the Deepgram TTS service. + + Args: + api_key: Deepgram API key for authentication. + voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en". + base_url: Custom base URL for Deepgram API. Uses default if empty. + sample_rate: Audio sample rate in Hz. If None, uses service default. + encoding: Audio encoding format. Defaults to "linear16". + **kwargs: Additional arguments passed to parent TTSService class. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._settings = { @@ -48,10 +71,23 @@ class DeepgramTTSService(TTSService): self._deepgram_client = DeepgramClient(api_key, config=client_options) def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True, as Deepgram TTS service supports metrics generation. + """ return True @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Deepgram's TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech, plus start/stop frames. + """ logger.debug(f"{self}: Generating TTS [{text}]") options = SpeakOptions( diff --git a/src/pipecat/services/deepseek/llm.py b/src/pipecat/services/deepseek/llm.py index aec6c50ba..55aaa341f 100644 --- a/src/pipecat/services/deepseek/llm.py +++ b/src/pipecat/services/deepseek/llm.py @@ -21,12 +21,6 @@ class DeepSeekLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to DeepSeek's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing DeepSeek's API. - base_url: The base URL for DeepSeek API. Defaults to "https://api.deepseek.com/v1". - model: The model identifier to use. Defaults to "deepseek-chat". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -37,6 +31,14 @@ class DeepSeekLLMService(OpenAILLMService): model: str = "deepseek-chat", **kwargs, ): + """Initialize the DeepSeek LLM service. + + Args: + api_key: The API key for accessing DeepSeek's API. + base_url: The base URL for DeepSeek API. Defaults to "https://api.deepseek.com/v1". + model: The model identifier to use. Defaults to "deepseek-chat". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index 2e04bd1b1..7153c39c5 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""ElevenLabs text-to-speech service implementations. + +This module provides WebSocket and HTTP-based TTS services using ElevenLabs API +with support for streaming audio, word timestamps, and voice customization. +""" + import asyncio import base64 import json @@ -57,6 +63,14 @@ ELEVENLABS_MULTILINGUAL_MODELS = { def language_to_elevenlabs_language(language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding ElevenLabs language code, or None if not supported. + """ BASE_LANGUAGES = { Language.AR: "ar", Language.BG: "bg", @@ -106,6 +120,14 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]: def output_format_from_sample_rate(sample_rate: int) -> str: + """Get the appropriate output format string for a given sample rate. + + Args: + sample_rate: The audio sample rate in Hz. + + Returns: + The ElevenLabs output format string. + """ match sample_rate: case 8000: return "pcm_8000" @@ -129,10 +151,10 @@ def build_elevenlabs_voice_settings( """Build voice settings dictionary for ElevenLabs based on provided settings. Args: - settings: Dictionary containing voice settings parameters + settings: Dictionary containing voice settings parameters. Returns: - Dictionary of voice settings or None if no valid settings are provided + Dictionary of voice settings or None if no valid settings are provided. """ voice_setting_keys = ["stability", "similarity_boost", "style", "use_speaker_boost", "speed"] @@ -147,6 +169,15 @@ def build_elevenlabs_voice_settings( def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float ) -> List[Tuple[str, float]]: + """Calculate word timestamps from character alignment information. + + Args: + alignment_info: Character alignment data from ElevenLabs API. + cumulative_time: Base time offset for this chunk. + + Returns: + List of (word, timestamp) tuples. + """ zipped_times = list(zip(alignment_info["chars"], alignment_info["charStartTimesMs"])) words = "".join(alignment_info["chars"]).split(" ") @@ -166,7 +197,28 @@ def calculate_word_times( class ElevenLabsTTSService(AudioContextWordTTSService): + """ElevenLabs WebSocket-based TTS service with word timestamps. + + Provides real-time text-to-speech using ElevenLabs' WebSocket streaming API. + Supports word-level timestamps, audio context management, and various voice + customization options including stability, similarity boost, and speed controls. + """ + class InputParams(BaseModel): + """Input parameters for ElevenLabs TTS configuration. + + Parameters: + language: Language to use for synthesis. + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.25 to 4.0). + auto_mode: Whether to enable automatic mode optimization. + enable_ssml_parsing: Whether to parse SSML tags in text. + enable_logging: Whether to enable ElevenLabs logging. + """ + language: Optional[Language] = None stability: Optional[float] = None similarity_boost: Optional[float] = None @@ -188,6 +240,17 @@ class ElevenLabsTTSService(AudioContextWordTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the ElevenLabs TTS service. + + Args: + api_key: ElevenLabs API key for authentication. + voice_id: ID of the voice to use for synthesis. + model: TTS model to use (e.g., "eleven_flash_v2_5"). + url: WebSocket URL for ElevenLabs TTS API. + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent service. + """ # Aggregating sentences still gives cleaner-sounding results and fewer # artifacts than streaming one word at a time. On average, waiting for a # full sentence should only "cost" us 15ms or so with GPT-4o or a Llama @@ -244,21 +307,40 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._keepalive_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as ElevenLabs service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs language format. + + Args: + language: The language to convert. + + Returns: + The ElevenLabs-specific language code, or None if not supported. + """ return language_to_elevenlabs_language(language) def _set_voice_settings(self): return build_elevenlabs_voice_settings(self._settings) async def set_model(self, model: str): + """Set the TTS model and reconnect. + + Args: + model: The model name to use for synthesis. + """ await super().set_model(model) logger.info(f"Switching TTS model to: [{model}]") await self._disconnect() await self._connect() async def _update_settings(self, settings: Mapping[str, Any]): + """Update service settings and reconnect if voice changed.""" prev_voice = self._voice_id await super()._update_settings(settings) if not prev_voice == self._voice_id: @@ -267,19 +349,35 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._connect() async def start(self, frame: StartFrame): + """Start the ElevenLabs TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._output_format = output_format_from_sample_rate(self.sample_rate) await self._connect() async def stop(self, frame: EndFrame): + """Stop the ElevenLabs TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the ElevenLabs TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def flush_audio(self): + """Flush any pending audio and finalize the current context.""" if not self._context_id or not self._websocket: return logger.trace(f"{self}: flushing audio") @@ -287,6 +385,12 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._websocket.send(json.dumps(msg)) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False @@ -374,6 +478,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): raise Exception("Websocket not connected") async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + """Handle interruption by closing the current context.""" await super()._handle_interruption(frame, direction) # Close the current context when interrupted without closing the websocket @@ -395,6 +500,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._started = False async def _receive_messages(self): + """Handle incoming WebSocket messages from ElevenLabs.""" async for message in WatchdogAsyncIterator( self._get_websocket(), manager=self.task_manager ): @@ -428,6 +534,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._cumulative_time = word_times[-1][1] async def _keepalive_task_handler(self): + """Send periodic keepalive messages to maintain WebSocket connection.""" KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3 while True: self.reset_watchdog() @@ -453,12 +560,21 @@ class ElevenLabsTTSService(AudioContextWordTTSService): break async def _send_text(self, text: str): + """Send text to the WebSocket for synthesis.""" if self._websocket and self._context_id: msg = {"text": text, "context_id": self._context_id} await self._websocket.send(json.dumps(msg)) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using ElevenLabs' streaming WebSocket API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -497,19 +613,26 @@ class ElevenLabsTTSService(AudioContextWordTTSService): class ElevenLabsHttpTTSService(WordTTSService): - """ElevenLabs Text-to-Speech service using HTTP streaming with word timestamps. + """ElevenLabs HTTP-based TTS service with word timestamps. - Args: - api_key: ElevenLabs API key - voice_id: ID of the voice to use - aiohttp_session: aiohttp ClientSession - model: Model ID (default: "eleven_flash_v2_5" for low latency) - base_url: API base URL - sample_rate: Output sample rate - params: Additional parameters for voice configuration + Provides text-to-speech using ElevenLabs' HTTP streaming API for simpler, + non-WebSocket integration. Suitable for use cases where streaming WebSocket + connection is not required or desired. """ class InputParams(BaseModel): + """Input parameters for ElevenLabs HTTP TTS configuration. + + Parameters: + language: Language to use for synthesis. + optimize_streaming_latency: Latency optimization level (0-4). + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.25 to 4.0). + """ + language: Optional[Language] = None optimize_streaming_latency: Optional[int] = None stability: Optional[float] = None @@ -530,6 +653,18 @@ class ElevenLabsHttpTTSService(WordTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the ElevenLabs HTTP TTS service. + + Args: + api_key: ElevenLabs API key for authentication. + voice_id: ID of the voice to use for synthesis. + aiohttp_session: aiohttp ClientSession for HTTP requests. + model: TTS model to use (e.g., "eleven_flash_v2_5"). + base_url: Base URL for ElevenLabs HTTP API. + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent service. + """ super().__init__( aggregate_sentences=True, push_text_frames=False, @@ -569,11 +704,22 @@ class ElevenLabsHttpTTSService(WordTTSService): self._previous_text = "" def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert pipecat Language to ElevenLabs language code.""" + """Convert pipecat Language to ElevenLabs language code. + + Args: + language: The language to convert. + + Returns: + The ElevenLabs-specific language code, or None if not supported. + """ return language_to_elevenlabs_language(language) def can_generate_metrics(self) -> bool: - """Indicate that this service can generate usage metrics.""" + """Check if this service can generate processing metrics. + + Returns: + True, as ElevenLabs HTTP service supports metrics generation. + """ return True def _set_voice_settings(self): @@ -587,12 +733,22 @@ class ElevenLabsHttpTTSService(WordTTSService): logger.debug(f"{self}: Reset internal state") async def start(self, frame: StartFrame): - """Initialize the service upon receiving a StartFrame.""" + """Start the ElevenLabs HTTP TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._output_format = output_format_from_sample_rate(self.sample_rate) self._reset_state() async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (StartInterruptionFrame, TTSStoppedFrame)): # Reset timing on interruption or stop @@ -619,10 +775,10 @@ class ElevenLabsHttpTTSService(WordTTSService): [("Hello", 0.1), ("world", 0.5)] Args: - alignment_info: Character timing data from ElevenLabs + alignment_info: Character timing data from ElevenLabs. Returns: - List of (word, timestamp) pairs + List of (word, timestamp) pairs. """ chars = alignment_info.get("characters", []) char_start_times = alignment_info.get("character_start_times_seconds", []) @@ -673,10 +829,10 @@ class ElevenLabsHttpTTSService(WordTTSService): Includes previous text as context for better prosody continuity. Args: - text: Text to convert to speech + text: Text to convert to speech. Yields: - Audio and control frames + Frame: Audio and control frames containing the synthesized speech. """ logger.debug(f"{self}: Generating TTS [{text}]") diff --git a/src/pipecat/services/fal/image.py b/src/pipecat/services/fal/image.py index 78439486e..c110ec3be 100644 --- a/src/pipecat/services/fal/image.py +++ b/src/pipecat/services/fal/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Fal's image generation service implementation. + +This module provides integration with Fal's image generation API +for creating images from text prompts using various AI models. +""" + import asyncio import io import os @@ -26,7 +32,25 @@ except ModuleNotFoundError as e: class FalImageGenService(ImageGenService): + """Fal's image generation service. + + Provides text-to-image generation using Fal.ai's API with configurable + parameters for image quality, safety, and format options. + """ + class InputParams(BaseModel): + """Input parameters for Fal.ai image generation. + + Parameters: + seed: Random seed for reproducible generation. If None, uses random seed. + num_inference_steps: Number of inference steps for generation. Defaults to 8. + num_images: Number of images to generate. Defaults to 1. + image_size: Image dimensions as string preset or dict with width/height. Defaults to "square_hd". + expand_prompt: Whether to automatically expand/enhance the prompt. Defaults to False. + enable_safety_checker: Whether to enable content safety filtering. Defaults to True. + format: Output image format. Defaults to "png". + """ + seed: Optional[int] = None num_inference_steps: int = 8 num_images: int = 1 @@ -44,6 +68,15 @@ class FalImageGenService(ImageGenService): key: Optional[str] = None, **kwargs, ): + """Initialize the FalImageGenService. + + Args: + params: Input parameters for image generation configuration. + aiohttp_session: HTTP client session for downloading generated images. + model: The Fal.ai model to use for generation. Defaults to "fal-ai/fast-sdxl". + key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable. + **kwargs: Additional arguments passed to parent ImageGenService. + """ super().__init__(**kwargs) self.set_model_name(model) self._params = params @@ -52,6 +85,16 @@ class FalImageGenService(ImageGenService): os.environ["FAL_KEY"] = key async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate an image from a text prompt. + + Args: + prompt: The text prompt to generate an image from. + + Yields: + URLImageRawFrame: Frame containing the generated image data and metadata. + ErrorFrame: If image generation fails. + """ + def load_image_bytes(encoded_image: bytes): buffer = io.BytesIO(encoded_image) image = Image.open(buffer) diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index 1e26d9958..3485a7de1 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Fal speech-to-text service implementation. + +This module provides integration with Fal's Wizper API for speech-to-text +transcription using segmented audio processing. +""" + import os from typing import AsyncGenerator, Optional @@ -27,7 +33,14 @@ except ModuleNotFoundError as e: def language_to_fal_language(language: Language) -> Optional[str]: - """Language support for Fal's Wizper API.""" + """Convert a Language enum to Fal's Wizper language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Fal Wizper language code, or None if not supported. + """ BASE_LANGUAGES = { Language.AF: "af", Language.AM: "am", @@ -145,18 +158,12 @@ class FalSTTService(SegmentedSTTService): This service uses Fal's Wizper API to perform speech-to-text transcription on audio segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. - - Args: - api_key: Fal API key. If not provided, will check FAL_KEY environment variable. - sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. - params: Configuration parameters for the Wizper API. - **kwargs: Additional arguments passed to SegmentedSTTService. """ class InputParams(BaseModel): """Configuration parameters for Fal's Wizper API. - Attributes: + Parameters: language: Language of the audio input. Defaults to English. task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'. chunk_level: Level of chunking ('segment'). Defaults to 'segment'. @@ -176,6 +183,14 @@ class FalSTTService(SegmentedSTTService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the FalSTTService with API key and parameters. + + Args: + api_key: Fal API key. If not provided, will check FAL_KEY environment variable. + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. + params: Configuration parameters for the Wizper API. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ super().__init__( sample_rate=sample_rate, **kwargs, @@ -201,16 +216,39 @@ class FalSTTService(SegmentedSTTService): } def can_generate_metrics(self) -> bool: + """Check if the service can generate processing metrics. + + Returns: + True, as Fal STT service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Fal's service-specific language code. + + Args: + language: The language to convert. + + Returns: + The Fal-specific language code, or None if not supported. + """ return language_to_fal_language(language) async def set_language(self, language: Language): + """Set the transcription language. + + Args: + language: The language to use for speech-to-text transcription. + """ logger.info(f"Switching STT language to: [{language}]") self._settings["language"] = self.language_to_service_language(language) async def set_model(self, model: str): + """Set the STT model. + + Args: + model: The model name to use for transcription. + """ await super().set_model(model) logger.info(f"Switching STT model to: [{model}]") @@ -229,7 +267,7 @@ class FalSTTService(SegmentedSTTService): audio: Raw audio bytes in WAV format (already converted by base class). Yields: - Frame: TranscriptionFrame containing the transcribed text. + Frame: TranscriptionFrame containing the transcribed text, or ErrorFrame on failure. Note: The audio is already in WAV format from the SegmentedSTTService. diff --git a/src/pipecat/services/fireworks/llm.py b/src/pipecat/services/fireworks/llm.py index cccfb5556..9edd6215c 100644 --- a/src/pipecat/services/fireworks/llm.py +++ b/src/pipecat/services/fireworks/llm.py @@ -20,12 +20,6 @@ class FireworksLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Fireworks' API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing Fireworks AI. - model: The model identifier to use. Defaults to "accounts/fireworks/models/firefunction-v2". - base_url: The base URL for Fireworks API. Defaults to "https://api.fireworks.ai/inference/v1". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -36,6 +30,14 @@ class FireworksLLMService(OpenAILLMService): base_url: str = "https://api.fireworks.ai/inference/v1", **kwargs, ): + """Initialize the Fireworks LLM service. + + Args: + api_key: The API key for accessing Fireworks AI. + model: The model identifier to use. Defaults to "accounts/fireworks/models/firefunction-v2". + base_url: The base URL for Fireworks API. Defaults to "https://api.fireworks.ai/inference/v1". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 84e285aee..7f2b85bdb 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Fish Audio text-to-speech service implementation. + +This module provides integration with Fish Audio's real-time TTS WebSocket API +for streaming text-to-speech synthesis with customizable voice parameters. +""" + import uuid from typing import AsyncGenerator, Literal, Optional @@ -39,7 +45,23 @@ FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] class FishAudioTTSService(InterruptibleTTSService): + """Fish Audio text-to-speech service with WebSocket streaming. + + Provides real-time text-to-speech synthesis using Fish Audio's WebSocket API. + Supports various audio formats, customizable prosody controls, and streaming + audio generation with interruption handling. + """ + class InputParams(BaseModel): + """Input parameters for Fish Audio TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + latency: Latency mode ("normal" or "balanced"). Defaults to "normal". + prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0. + prosody_volume: Volume adjustment in dB. Defaults to 0. + """ + language: Optional[Language] = Language.EN latency: Optional[str] = "normal" # "normal" or "balanced" prosody_speed: Optional[float] = 1.0 # Speech speed (0.5-2.0) @@ -55,6 +77,16 @@ class FishAudioTTSService(InterruptibleTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Fish Audio TTS service. + + Args: + api_key: Fish Audio API key for authentication. + model: Reference ID of the voice model to use for synthesis. + output_format: Audio output format. Defaults to "pcm". + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent service. + """ super().__init__( push_stop_frames=True, pause_frame_processing=True, @@ -85,23 +117,48 @@ class FishAudioTTSService(InterruptibleTTSService): self.set_model_name(model) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Fish Audio service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the TTS model (reference ID). + + Args: + model: The reference ID of the voice model to use. + """ self._settings["reference_id"] = model await super().set_model(model) logger.info(f"Switching TTS model to: [{model}]") async def start(self, frame: StartFrame): + """Start the Fish Audio TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["sample_rate"] = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): + """Stop the Fish Audio TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the Fish Audio TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() @@ -191,6 +248,14 @@ class FishAudioTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Fish Audio's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames and control frames for the synthesized speech. + """ logger.debug(f"{self}: Generating Fish TTS: [{text}]") try: if not self._websocket or self._websocket.closed: diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index afaf966dc..3c5ed92dc 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -333,7 +333,12 @@ class GeminiMultimodalLiveContextAggregatorPair: class GeminiMultimodalModalities(Enum): - """Supported modalities for Gemini Multimodal Live.""" + """Supported modalities for Gemini Multimodal Live. + + Parameters: + TEXT: Text responses. + AUDIO: Audio responses. + """ TEXT = "TEXT" AUDIO = "AUDIO" @@ -422,20 +427,6 @@ class GeminiMultimodalLiveLLMService(LLMService): This service enables real-time conversations with Gemini, supporting both text and audio modalities. It handles voice transcription, streaming audio responses, and tool usage. - - Args: - api_key: Google AI API key for authentication. - base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint. - model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001". - voice_id: TTS voice identifier. Defaults to "Charon". - start_audio_paused: Whether to start with audio input paused. Defaults to False. - start_video_paused: Whether to start with video input paused. Defaults to False. - system_instruction: System prompt for the model. Defaults to None. - tools: Tools/functions available to the model. Defaults to None. - params: Configuration parameters for the model. Defaults to InputParams(). - inference_on_context_initialization: Whether to generate a response when context - is first set. Defaults to True. - **kwargs: Additional arguments passed to parent LLMService. """ # Overriding the default adapter to use the Gemini one. @@ -456,6 +447,22 @@ class GeminiMultimodalLiveLLMService(LLMService): inference_on_context_initialization: bool = True, **kwargs, ): + """Initialize the Gemini Multimodal Live LLM service. + + Args: + api_key: Google AI API key for authentication. + base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint. + model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001". + voice_id: TTS voice identifier. Defaults to "Charon". + start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_video_paused: Whether to start with video input paused. Defaults to False. + system_instruction: System prompt for the model. Defaults to None. + tools: Tools/functions available to the model. Defaults to None. + params: Configuration parameters for the model. Defaults to InputParams(). + inference_on_context_initialization: Whether to generate a response when context + is first set. Defaults to True. + **kwargs: Additional arguments passed to parent LLMService. + """ super().__init__(base_url=base_url, **kwargs) params = params or InputParams() diff --git a/src/pipecat/services/gladia/config.py b/src/pipecat/services/gladia/config.py index 662996f1d..0af008773 100644 --- a/src/pipecat/services/gladia/config.py +++ b/src/pipecat/services/gladia/config.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Configuration for the Gladia STT service.""" + from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel @@ -14,7 +16,7 @@ from pipecat.transcriptions.language import Language class LanguageConfig(BaseModel): """Configuration for language detection and handling. - Attributes: + Parameters: languages: List of language codes to use for transcription code_switching: Whether to auto-detect language changes during transcription """ @@ -26,7 +28,7 @@ class LanguageConfig(BaseModel): class PreProcessingConfig(BaseModel): """Configuration for audio pre-processing options. - Attributes: + Parameters: speech_threshold: Sensitivity for speech detection (0-1) """ @@ -36,7 +38,7 @@ class PreProcessingConfig(BaseModel): class CustomVocabularyItem(BaseModel): """Represents a custom vocabulary item with an intensity value. - Attributes: + Parameters: value: The vocabulary word or phrase intensity: The bias intensity for this vocabulary item (0-1) """ @@ -48,7 +50,7 @@ class CustomVocabularyItem(BaseModel): class CustomVocabularyConfig(BaseModel): """Configuration for custom vocabulary. - Attributes: + Parameters: vocabulary: List of words/phrases or CustomVocabularyItem objects default_intensity: Default intensity for simple string vocabulary items """ @@ -60,7 +62,7 @@ class CustomVocabularyConfig(BaseModel): class CustomSpellingConfig(BaseModel): """Configuration for custom spelling rules. - Attributes: + Parameters: spelling_dictionary: Mapping of correct spellings to phonetic variations """ @@ -70,7 +72,7 @@ class CustomSpellingConfig(BaseModel): class TranslationConfig(BaseModel): """Configuration for real-time translation. - Attributes: + Parameters: target_languages: List of target language codes for translation model: Translation model to use ("base" or "enhanced") match_original_utterances: Whether to align translations with original utterances @@ -92,7 +94,7 @@ class TranslationConfig(BaseModel): class RealtimeProcessingConfig(BaseModel): """Configuration for real-time processing features. - Attributes: + Parameters: words_accurate_timestamps: Whether to provide per-word timestamps custom_vocabulary: Whether to enable custom vocabulary custom_vocabulary_config: Custom vocabulary configuration @@ -118,7 +120,7 @@ class RealtimeProcessingConfig(BaseModel): class MessagesConfig(BaseModel): """Configuration for controlling which message types are sent via WebSocket. - Attributes: + Parameters: receive_partial_transcripts: Whether to receive intermediate transcription results receive_final_transcripts: Whether to receive final transcription results receive_speech_events: Whether to receive speech begin/end events @@ -144,7 +146,7 @@ class MessagesConfig(BaseModel): class GladiaInputParams(BaseModel): """Configuration parameters for the Gladia STT service. - Attributes: + Parameters: encoding: Audio encoding format bit_depth: Audio bit depth channels: Number of audio channels diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index 0aa144fda..c436a7ea9 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Gladia Speech-to-Text (STT) service implementation. + +This module provides a Speech-to-Text service using Gladia's real-time WebSocket API, +supporting multiple languages, custom vocabulary, and various audio processing options. +""" + import asyncio import base64 import json @@ -41,10 +47,10 @@ def language_to_gladia_language(language: Language) -> Optional[str]: """Convert a Language enum to Gladia's language code format. Args: - language: The Language enum value to convert + language: The Language enum value to convert. Returns: - The Gladia language code string or None if not supported + The Gladia language code string or None if not supported. """ BASE_LANGUAGES = { Language.AF: "af", @@ -180,6 +186,7 @@ class GladiaSTTService(STTService): This service connects to Gladia's WebSocket API for real-time transcription with support for multiple languages, custom vocabulary, and various processing options. + Provides automatic reconnection, audio buffering, and comprehensive error handling. For complete API documentation, see: https://docs.gladia.io/api-reference/v2/live/init """ @@ -204,16 +211,16 @@ class GladiaSTTService(STTService): """Initialize the Gladia STT service. Args: - api_key: Gladia API key - url: Gladia API URL - confidence: Minimum confidence threshold for transcriptions - sample_rate: Audio sample rate in Hz - model: Model to use ("solaria-1") - params: Additional configuration parameters - max_reconnection_attempts: Maximum number of reconnection attempts - reconnection_delay: Initial delay between reconnection attempts (exponential backoff) - max_buffer_size: Maximum size of audio buffer in bytes - **kwargs: Additional arguments passed to the STTService + api_key: Gladia API key for authentication. + url: Gladia API URL. Defaults to "https://api.gladia.io/v2/live". + confidence: Minimum confidence threshold for transcriptions (0.0-1.0). + sample_rate: Audio sample rate in Hz. If None, uses service default. + model: Model to use for transcription. Defaults to "solaria-1". + params: Additional configuration parameters for Gladia service. + max_reconnection_attempts: Maximum number of reconnection attempts. Defaults to 5. + reconnection_delay: Initial delay between reconnection attempts in seconds. + max_buffer_size: Maximum size of audio buffer in bytes. Defaults to 20MB. + **kwargs: Additional arguments passed to the STTService parent class. """ super().__init__(sample_rate=sample_rate, **kwargs) @@ -256,10 +263,22 @@ class GladiaSTTService(STTService): self._should_reconnect = True def can_generate_metrics(self) -> bool: + """Check if the service can generate performance metrics. + + Returns: + True, indicating this service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert pipecat Language enum to Gladia's language code.""" + """Convert pipecat Language enum to Gladia's language code. + + Args: + language: The Language enum value to convert. + + Returns: + The Gladia language code string or None if not supported. + """ return language_to_gladia_language(language) def _prepare_settings(self) -> Dict[str, Any]: @@ -314,7 +333,11 @@ class GladiaSTTService(STTService): return settings async def start(self, frame: StartFrame): - """Start the Gladia STT websocket connection.""" + """Start the Gladia STT websocket connection. + + Args: + frame: The start frame triggering service startup. + """ await super().start(frame) if self._connection_task: return @@ -323,7 +346,11 @@ class GladiaSTTService(STTService): self._connection_task = self.create_task(self._connection_handler()) async def stop(self, frame: EndFrame): - """Stop the Gladia STT websocket connection.""" + """Stop the Gladia STT websocket connection. + + Args: + frame: The end frame triggering service shutdown. + """ await super().stop(frame) self._should_reconnect = False await self._send_stop_recording() @@ -335,7 +362,11 @@ class GladiaSTTService(STTService): await self._cleanup_connection() async def cancel(self, frame: CancelFrame): - """Cancel the Gladia STT websocket connection.""" + """Cancel the Gladia STT websocket connection. + + Args: + frame: The cancel frame triggering service cancellation. + """ await super().cancel(frame) self._should_reconnect = False @@ -346,7 +377,14 @@ class GladiaSTTService(STTService): await self._cleanup_connection() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Run speech-to-text on audio data.""" + """Run speech-to-text on audio data. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None (processing is handled asynchronously via WebSocket). + """ await self.start_ttfb_metrics() await self.start_processing_metrics() diff --git a/src/pipecat/services/google/frames.py b/src/pipecat/services/google/frames.py index 700a39a7e..bc174937a 100644 --- a/src/pipecat/services/google/frames.py +++ b/src/pipecat/services/google/frames.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google AI service frames for search and grounding functionality. + +This module defines specialized frame types for handling search results +and grounding metadata from Google AI models, particularly for Gemini +models that support web search and fact grounding capabilities. +""" + from dataclasses import dataclass, field from typing import List, Optional @@ -12,12 +19,27 @@ from pipecat.frames.frames import DataFrame @dataclass class LLMSearchResult: + """Represents a single search result with confidence scores. + + Parameters: + text: The search result text content. + confidence: List of confidence scores associated with the result. + """ + text: str confidence: List[float] = field(default_factory=list) @dataclass class LLMSearchOrigin: + """Represents the origin source of search results. + + Parameters: + site_uri: URI of the source website. + site_title: Title of the source website. + results: List of search results from this origin. + """ + site_uri: Optional[str] = None site_title: Optional[str] = None results: List[LLMSearchResult] = field(default_factory=list) @@ -25,9 +47,27 @@ class LLMSearchOrigin: @dataclass class LLMSearchResponseFrame(DataFrame): + """Frame containing search results and grounding information from Google AI models. + + This frame is used to convey search results and grounding metadata + from Google AI models that support web search capabilities. It includes + the search result text, rendered content, and detailed origin information + with confidence scores. + + Parameters: + search_result: The main search result text. + rendered_content: Rendered content from the search entry point. + origins: List of search result origins with detailed information. + """ + search_result: Optional[str] = None rendered_content: Optional[str] = None origins: List[LLMSearchOrigin] = field(default_factory=list) def __str__(self): + """Return string representation of the search response frame. + + Returns: + String representation showing search result and origins. + """ return f"LLMSearchResponseFrame(search_result={self.search_result}, origins={self.origins})" diff --git a/src/pipecat/services/google/image.py b/src/pipecat/services/google/image.py index dc0218b8c..5d7a461b3 100644 --- a/src/pipecat/services/google/image.py +++ b/src/pipecat/services/google/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google AI image generation service implementation. + +This module provides integration with Google's Imagen model for generating +images from text prompts using the Google AI API. +""" + import io import os @@ -29,7 +35,22 @@ except ModuleNotFoundError as e: class GoogleImageGenService(ImageGenService): + """Google AI image generation service using Imagen models. + + Provides text-to-image generation capabilities using Google's Imagen models + through the Google AI API. Supports multiple image generation and negative + prompting for enhanced control over generated content. + """ + class InputParams(BaseModel): + """Configuration parameters for Google image generation. + + Parameters: + number_of_images: Number of images to generate (1-8). Defaults to 1. + model: Google Imagen model to use. Defaults to "imagen-3.0-generate-002". + negative_prompt: Optional negative prompt to guide what not to include. + """ + number_of_images: int = Field(default=1, ge=1, le=8) model: str = Field(default="imagen-3.0-generate-002") negative_prompt: Optional[str] = Field(default=None) @@ -41,22 +62,38 @@ class GoogleImageGenService(ImageGenService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the GoogleImageGenService with API key and parameters. + + Args: + api_key: Google AI API key for authentication. + params: Configuration parameters for image generation. Defaults to InputParams(). + **kwargs: Additional arguments passed to the parent ImageGenService. + """ super().__init__(**kwargs) self._params = params or GoogleImageGenService.InputParams() self._client = genai.Client(api_key=api_key) self.set_model_name(self._params.model) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Google image generation service supports metrics. + """ return True async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: """Generate images from a text prompt using Google's Imagen model. Args: - prompt (str): The text description to generate images from. + prompt: The text description to generate images from. Yields: - Frame: Generated image frames or error frames. + Frame: Generated URLImageRawFrame objects containing the generated + images, or ErrorFrame objects if generation fails. + + Raises: + Exception: If there are issues with the Google AI API or image processing. """ logger.debug(f"Generating image from prompt: {prompt}") await self.start_ttfb_metrics() diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index 6b8f51f33..bd56b8416 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -233,11 +233,6 @@ class GoogleLLMContext(OpenAILLMContext): This class handles conversion between OpenAI-style messages and Google AI's Content/Part format, including system messages, function calls, and media. - - Args: - messages: Initial messages in OpenAI format. - tools: Available tools/functions for the model. - tool_choice: Tool choice configuration. """ def __init__( @@ -246,6 +241,13 @@ class GoogleLLMContext(OpenAILLMContext): tools: Optional[List[dict]] = None, tool_choice: Optional[dict] = None, ): + """Initialize GoogleLLMContext. + + Args: + messages: Initial messages in OpenAI format. + tools: Available tools/functions for the model. + tool_choice: Tool choice configuration. + """ super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) self.system_message = None @@ -563,15 +565,6 @@ class GoogleLLMService(LLMService): from OpenAILLMContext to the messages format expected by the Google AI model. We use OpenAILLMContext as a lingua franca for all LLM services to enable easy switching between different LLMs. - - Args: - api_key: Google AI API key for authentication. - model: Model name to use. Defaults to "gemini-2.0-flash". - params: Input parameters for the model. - system_instruction: System instruction/prompt for the model. - tools: List of available tools/functions. - tool_config: Configuration for tool usage. - **kwargs: Additional arguments passed to parent class. """ # Overriding the default adapter to use the Gemini one. @@ -605,6 +598,17 @@ class GoogleLLMService(LLMService): tool_config: Optional[Dict[str, Any]] = None, **kwargs, ): + """Initialize the Google LLM service. + + Args: + api_key: Google AI API key for authentication. + model: Model name to use. Defaults to "gemini-2.0-flash". + params: Input parameters for the model. + system_instruction: System instruction/prompt for the model. + tools: List of available tools/functions. + tool_config: Configuration for tool usage. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) params = params or GoogleLLMService.InputParams() diff --git a/src/pipecat/services/google/llm_openai.py b/src/pipecat/services/google/llm_openai.py index 8677179c9..02c50f885 100644 --- a/src/pipecat/services/google/llm_openai.py +++ b/src/pipecat/services/google/llm_openai.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google LLM service using OpenAI-compatible API format. + +This module provides integration with Google's AI LLM models using the OpenAI +API format through Google's Gemini API OpenAI compatibility layer. +""" + import json import os @@ -25,8 +31,17 @@ from pipecat.services.openai.llm import OpenAILLMService class GoogleLLMOpenAIBetaService(OpenAILLMService): - """This class implements inference with Google's AI LLM models using the OpenAI format. - Ref - https://ai.google.dev/gemini-api/docs/openai + """Google LLM service using OpenAI-compatible API format. + + This service provides access to Google's AI LLM models (like Gemini) through + the OpenAI API format. It handles streaming responses, function calls, and + tool usage while maintaining compatibility with OpenAI's interface. + + Note: This service includes a workaround for a Google API bug where function + call indices may be incorrectly set to None, resulting in empty function names. + + Reference: + https://ai.google.dev/gemini-api/docs/openai """ def __init__( @@ -37,6 +52,14 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): model: str = "gemini-2.0-flash", **kwargs, ): + """Initialize the Google LLM service. + + Args: + api_key: Google API key for authentication. + base_url: Base URL for Google's OpenAI-compatible API. + model: Google model name to use (e.g., "gemini-2.0-flash"). + **kwargs: Additional arguments passed to the parent OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) async def _process_context(self, context: OpenAILLMContext): diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py index 9b0ed6501..22b6258a5 100644 --- a/src/pipecat/services/google/llm_vertex.py +++ b/src/pipecat/services/google/llm_vertex.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google Vertex AI LLM service implementation. + +This module provides integration with Google's AI models via Vertex AI while +maintaining OpenAI API compatibility through Google's OpenAI-compatible endpoint. +""" + import json import os @@ -31,16 +37,24 @@ except ModuleNotFoundError as e: class GoogleVertexLLMService(OpenAILLMService): - """Implements inference with Google's AI models via Vertex AI while - maintaining OpenAI API compatibility. + """Google Vertex AI LLM service with OpenAI API compatibility. + + Provides access to Google's AI models via Vertex AI while maintaining + OpenAI API compatibility. Handles authentication using Google service + account credentials and constructs appropriate endpoint URLs for + different GCP regions and projects. Reference: - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library - + https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library """ class InputParams(OpenAILLMService.InputParams): - """Input parameters specific to Vertex AI.""" + """Input parameters specific to Vertex AI. + + Parameters: + location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + project_id: Google Cloud project ID. + """ # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations location: str = "us-east4" @@ -58,11 +72,11 @@ class GoogleVertexLLMService(OpenAILLMService): """Initializes the VertexLLMService. Args: - credentials (Optional[str]): JSON string of service account credentials. - credentials_path (Optional[str]): Path to the service account JSON file. - model (str): Model identifier. Defaults to "google/gemini-2.0-flash-001". - params (InputParams): Vertex AI input parameters. - **kwargs: Additional arguments for OpenAILLMService. + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. + model: Model identifier (e.g., "google/gemini-2.0-flash-001"). + params: Vertex AI input parameters including location and project. + **kwargs: Additional arguments passed to OpenAILLMService. """ params = params or OpenAILLMService.InputParams() base_url = self._get_base_url(params) @@ -74,7 +88,7 @@ class GoogleVertexLLMService(OpenAILLMService): @staticmethod def _get_base_url(params: InputParams) -> str: - """Constructs the base URL for Vertex AI API.""" + """Construct the base URL for Vertex AI API.""" return ( f"https://{params.location}-aiplatform.googleapis.com/v1/" f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" @@ -82,14 +96,22 @@ class GoogleVertexLLMService(OpenAILLMService): @staticmethod def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: - """Retrieves an authentication token using Google service account credentials. + """Retrieve an authentication token using Google service account credentials. + + Supports multiple authentication methods: + 1. Direct JSON credentials string + 2. Path to service account JSON file + 3. Default application credentials (ADC) Args: - credentials (Optional[str]): JSON string of service account credentials. - credentials_path (Optional[str]): Path to the service account JSON file. + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. Returns: - str: OAuth token for API authentication. + OAuth token for API authentication. + + Raises: + ValueError: If no valid credentials are provided or found. """ creds: Optional[service_account.Credentials] = None diff --git a/src/pipecat/services/google/rtvi.py b/src/pipecat/services/google/rtvi.py index cd60f6f1f..3031ad532 100644 --- a/src/pipecat/services/google/rtvi.py +++ b/src/pipecat/services/google/rtvi.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google RTVI integration models and observer implementation. + +This module provides integration with Google's services through the RTVI framework, +including models for search responses and an observer for handling Google-specific +frame types. +""" + from typing import List, Literal, Optional from pydantic import BaseModel @@ -16,22 +23,56 @@ from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFra class RTVISearchResponseMessageData(BaseModel): + """Data payload for search response messages in RTVI protocol. + + Parameters: + search_result: The search result text, if available. + rendered_content: The rendered content from the search, if available. + origins: List of search result origins with metadata. + """ + search_result: Optional[str] rendered_content: Optional[str] origins: List[LLMSearchOrigin] class RTVIBotLLMSearchResponseMessage(BaseModel): + """RTVI message for bot LLM search responses. + + Parameters: + label: Always "rtvi-ai" for RTVI protocol messages. + type: Always "bot-llm-search-response" for this message type. + data: The search response data payload. + """ + label: Literal["rtvi-ai"] = "rtvi-ai" type: Literal["bot-llm-search-response"] = "bot-llm-search-response" data: RTVISearchResponseMessageData class GoogleRTVIObserver(RTVIObserver): + """RTVI observer for Google service integration. + + Extends the base RTVIObserver to handle Google-specific frame types, + particularly LLM search response frames from Google services. + """ + def __init__(self, rtvi: RTVIProcessor): + """Initialize the Google RTVI observer. + + Args: + rtvi: The RTVI processor to send messages through. + """ super().__init__(rtvi) async def on_push_frame(self, data: FramePushed): + """Process frames being pushed through the pipeline. + + Handles Google-specific frames in addition to the base RTVI frame types. + + Args: + data: Frame push event data containing frame and metadata. + """ await super().on_push_frame(data) frame = data.frame diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 274aba2fa..e94fbbb12 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google Cloud Speech-to-Text V2 service implementation for Pipecat. + +This module provides a Google Cloud Speech-to-Text V2 service with streaming +support, enabling real-time speech recognition with features like automatic +punctuation, voice activity detection, and multi-language support. +""" + import asyncio import json import os @@ -353,9 +360,15 @@ class GoogleSTTService(STTService): Provides real-time speech recognition using Google Cloud's Speech-to-Text V2 API with streaming support. Handles audio transcription and optional voice activity detection. + Implements automatic stream reconnection to handle Google's 4-minute streaming limit. Attributes: InputParams: Configuration parameters for the STT service. + STREAMING_LIMIT: Google Cloud's streaming limit in milliseconds (4 minutes). + + Raises: + ValueError: If neither credentials nor credentials_path is provided. + ValueError: If project ID is not found in credentials. """ # Google Cloud's STT service has a connection time limit of 5 minutes per stream. @@ -367,7 +380,7 @@ class GoogleSTTService(STTService): class InputParams(BaseModel): """Configuration parameters for Google Speech-to-Text. - Attributes: + Parameters: languages: Single language or list of recognition languages. First language is primary. model: Speech recognition model to use. use_separate_recognition_per_channel: Process each audio channel separately. @@ -396,13 +409,25 @@ class GoogleSTTService(STTService): @field_validator("languages", mode="before") @classmethod def validate_languages(cls, v) -> List[Language]: + """Ensure languages is always a list. + + Args: + v: Single Language enum or list of Language enums. + + Returns: + List[Language]: List of configured languages. + """ if isinstance(v, Language): return [v] return v @property def language_list(self) -> List[Language]: - """Get languages as a guaranteed list.""" + """Get languages as a guaranteed list. + + Returns: + List[Language]: List of configured languages. + """ assert isinstance(self.languages, list) return self.languages @@ -425,10 +450,6 @@ class GoogleSTTService(STTService): sample_rate: Audio sample rate in Hertz. params: Configuration parameters for the service. **kwargs: Additional arguments passed to STTService. - - Raises: - ValueError: If neither credentials nor credentials_path is provided. - ValueError: If project ID is not found in credentials. """ super().__init__(sample_rate=sample_rate, **kwargs) @@ -501,6 +522,11 @@ class GoogleSTTService(STTService): } def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + bool: True, as this service supports metrics generation. + """ return True def language_to_service_language(self, language: Language | List[Language]) -> str | List[str]: @@ -548,7 +574,11 @@ class GoogleSTTService(STTService): await self._reconnect_if_needed() async def set_model(self, model: str): - """Update the service's recognition model.""" + """Update the service's recognition model. + + Args: + model: The new recognition model to use. + """ logger.debug(f"Switching STT model to: {model}") await super().set_model(model) self._settings["model"] = model @@ -556,14 +586,29 @@ class GoogleSTTService(STTService): await self._reconnect_if_needed() async def start(self, frame: StartFrame): + """Start the STT service and establish connection. + + Args: + frame: The start frame triggering the service start. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the STT service and clean up resources. + + Args: + frame: The end frame triggering the service stop. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the STT service and clean up resources. + + Args: + frame: The cancel frame triggering the service cancellation. + """ await super().cancel(frame) await self._disconnect() @@ -585,7 +630,7 @@ class GoogleSTTService(STTService): """Update service options dynamically. Args: - languages: New list of recongition languages. + languages: New list of recognition languages. model: New recognition model. enable_automatic_punctuation: Enable/disable automatic punctuation. enable_spoken_punctuation: Enable/disable spoken punctuation. @@ -767,7 +812,14 @@ class GoogleSTTService(STTService): await self.push_frame(ErrorFrame(str(e))) async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Process an audio chunk for STT transcription.""" + """Process an audio chunk for STT transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: None (actual transcription frames are pushed via internal processing). + """ if self._streaming_task: # Queue the audio data await self.start_ttfb_metrics() diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 6e57b7b8d..40de8afff 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -4,7 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio +"""Google Cloud Text-to-Speech service implementations. + +This module provides integration with Google Cloud Text-to-Speech API, +offering both HTTP-based synthesis with SSML support and streaming synthesis +for real-time applications. +""" + import json import os @@ -43,6 +49,14 @@ except ModuleNotFoundError as e: def language_to_google_tts_language(language: Language) -> Optional[str]: + """Convert a Language enum to Google TTS language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Google TTS language code, or None if not supported. + """ language_map = { # Afrikaans Language.AF: "af-ZA", @@ -203,7 +217,32 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: class GoogleHttpTTSService(TTSService): + """Google Cloud Text-to-Speech HTTP service with SSML support. + + Provides text-to-speech synthesis using Google Cloud's HTTP API with + comprehensive SSML support for voice customization, prosody control, + and styling options. Ideal for applications requiring fine-grained + control over speech output. + + Note: + Requires Google Cloud credentials via service account JSON, credentials file, + or default application credentials (GOOGLE_APPLICATION_CREDENTIALS). + Chirp and Journey voices don't support SSML and will use plain text input. + """ + class InputParams(BaseModel): + """Input parameters for Google HTTP TTS voice customization. + + Parameters: + pitch: Voice pitch adjustment (e.g., "+2st", "-50%"). + rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). + volume: Volume adjustment (e.g., "loud", "soft", "+6dB"). + emphasis: Emphasis level for the text. + language: Language for synthesis. Defaults to English. + gender: Voice gender preference. + google_style: Google-specific voice style. + """ + pitch: Optional[str] = None rate: Optional[str] = None volume: Optional[str] = None @@ -222,6 +261,16 @@ class GoogleHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initializes the Google HTTP TTS service. + + Args: + credentials: JSON string containing Google Cloud service account credentials. + credentials_path: Path to Google Cloud service account JSON file. + voice_id: Google TTS voice identifier (e.g., "en-US-Standard-A"). + sample_rate: Audio sample rate in Hz. If None, uses default. + params: Voice customization parameters including pitch, rate, volume, etc. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or GoogleHttpTTSService.InputParams() @@ -245,11 +294,20 @@ class GoogleHttpTTSService(TTSService): def _create_client( self, credentials: Optional[str], credentials_path: Optional[str] ) -> texttospeech_v1.TextToSpeechAsyncClient: + """Create authenticated Google Text-to-Speech client. + + Args: + credentials: JSON string with service account credentials. + credentials_path: Path to service account JSON file. + + Returns: + Authenticated TextToSpeechAsyncClient instance. + + Raises: + ValueError: If no valid credentials are provided. + """ creds: Optional[service_account.Credentials] = None - # Create a Google Cloud service account for the Cloud Text-to-Speech API - # Using either the provided credentials JSON string or the path to a service account JSON - # file, create a Google Cloud service account and use it to authenticate with the API. if credentials: # Use provided credentials JSON string json_account_info = json.loads(credentials) @@ -271,9 +329,22 @@ class GoogleHttpTTSService(TTSService): return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Google HTTP TTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Google TTS language format. + + Args: + language: The language to convert. + + Returns: + The Google TTS-specific language code, or None if not supported. + """ return language_to_google_tts_language(language) def _construct_ssml(self, text: str) -> str: @@ -324,6 +395,14 @@ class GoogleHttpTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Google's HTTP TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -381,19 +460,13 @@ class GoogleHttpTTSService(TTSService): class GoogleTTSService(TTSService): - """Text-to-Speech service using Google Cloud Text-to-Speech API. + """Google Cloud Text-to-Speech streaming service. - Converts text to speech using Google's TTS models with streaming synthesis - for low latency. Supports multiple languages and voices. + Provides real-time text-to-speech synthesis using Google Cloud's streaming API + for low-latency applications. Optimized for Chirp 3 HD and Journey voices + with continuous audio streaming capabilities. - Args: - credentials: JSON string containing Google Cloud service account credentials. - credentials_path: Path to Google Cloud service account JSON file. - voice_id: Google TTS voice identifier (e.g., "en-US-Chirp3-HD-Charon"). - sample_rate: Audio sample rate in Hz. - params: Language only. - - Notes: + Note: Requires Google Cloud credentials via service account JSON, file path, or default application credentials (GOOGLE_APPLICATION_CREDENTIALS env var). Only Chirp 3 HD and Journey voices are supported. Use GoogleHttpTTSService for other voices. @@ -411,6 +484,12 @@ class GoogleTTSService(TTSService): """ class InputParams(BaseModel): + """Input parameters for Google streaming TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + """ + language: Optional[Language] = Language.EN def __init__( @@ -423,6 +502,16 @@ class GoogleTTSService(TTSService): params: InputParams = InputParams(), **kwargs, ): + """Initializes the Google streaming TTS service. + + Args: + credentials: JSON string containing Google Cloud service account credentials. + credentials_path: Path to Google Cloud service account JSON file. + voice_id: Google TTS voice identifier (e.g., "en-US-Chirp3-HD-Charon"). + sample_rate: Audio sample rate in Hz. If None, uses default. + params: Language configuration parameters. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or GoogleTTSService.InputParams() @@ -466,13 +555,34 @@ class GoogleTTSService(TTSService): return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Google streaming TTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Google TTS language format. + + Args: + language: The language to convert. + + Returns: + The Google TTS-specific language code, or None if not supported. + """ return language_to_google_tts_language(language) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate streaming speech from text using Google's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech as it's generated. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/grok/llm.py b/src/pipecat/services/grok/llm.py index e7d817c5f..2a9704008 100644 --- a/src/pipecat/services/grok/llm.py +++ b/src/pipecat/services/grok/llm.py @@ -67,12 +67,6 @@ class GrokLLMService(OpenAILLMService): maintaining full compatibility with OpenAI's interface and functionality. Includes specialized token usage tracking that accumulates metrics during processing and reports final totals. - - Args: - api_key: The API key for accessing Grok's API. - base_url: The base URL for Grok API. Defaults to "https://api.x.ai/v1". - model: The model identifier to use. Defaults to "grok-3-beta". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -83,6 +77,14 @@ class GrokLLMService(OpenAILLMService): model: str = "grok-3-beta", **kwargs, ): + """Initialize the GrokLLMService with API key and model. + + Args: + api_key: The API key for accessing Grok's API. + base_url: The base URL for Grok API. Defaults to "https://api.x.ai/v1". + model: The model identifier to use. Defaults to "grok-3-beta". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) # Initialize counters for token usage metrics self._prompt_tokens = 0 diff --git a/src/pipecat/services/groq/llm.py b/src/pipecat/services/groq/llm.py index e7edb4996..57f2a533d 100644 --- a/src/pipecat/services/groq/llm.py +++ b/src/pipecat/services/groq/llm.py @@ -16,12 +16,6 @@ class GroqLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Groq's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing Groq's API. - base_url: The base URL for Groq API. Defaults to "https://api.groq.com/openai/v1". - model: The model identifier to use. Defaults to "llama-3.3-70b-versatile". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -32,6 +26,14 @@ class GroqLLMService(OpenAILLMService): model: str = "llama-3.3-70b-versatile", **kwargs, ): + """Initialize Groq LLM service. + + Args: + api_key: The API key for accessing Groq's API. + base_url: The base URL for Groq API. Defaults to "https://api.groq.com/openai/v1". + model: The model identifier to use. Defaults to "llama-3.3-70b-versatile". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): diff --git a/src/pipecat/services/groq/stt.py b/src/pipecat/services/groq/stt.py index 5852bedfd..1267dd85f 100644 --- a/src/pipecat/services/groq/stt.py +++ b/src/pipecat/services/groq/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Groq speech-to-text service implementation using Whisper models.""" + from typing import Optional from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -15,15 +17,6 @@ class GroqSTTService(BaseWhisperSTTService): Uses Groq's Whisper API to convert audio to text. Requires a Groq API key set via the api_key parameter or GROQ_API_KEY environment variable. - - Args: - model: Whisper model to use. Defaults to "whisper-large-v3-turbo". - api_key: Groq API key. Defaults to None. - base_url: API base URL. Defaults to "https://api.groq.com/openai/v1". - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to BaseWhisperSTTService. """ def __init__( @@ -37,6 +30,17 @@ class GroqSTTService(BaseWhisperSTTService): temperature: Optional[float] = None, **kwargs, ): + """Initialize Groq STT service. + + Args: + model: Whisper model to use. Defaults to "whisper-large-v3-turbo". + api_key: Groq API key. Defaults to None. + base_url: API base URL. Defaults to "https://api.groq.com/openai/v1". + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to BaseWhisperSTTService. + """ super().__init__( model=model, api_key=api_key, diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 33fd3ce34..68ba4a598 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Groq text-to-speech service implementation.""" + import io import wave from typing import AsyncGenerator, Optional @@ -25,7 +27,21 @@ except ModuleNotFoundError as e: class GroqTTSService(TTSService): + """Groq text-to-speech service implementation. + + Provides text-to-speech synthesis using Groq's TTS API. The service + operates at a fixed 48kHz sample rate and supports various voices + and output formats. + """ + class InputParams(BaseModel): + """Input parameters for Groq TTS configuration. + + Parameters: + language: Language for speech synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -42,6 +58,17 @@ class GroqTTSService(TTSService): sample_rate: Optional[int] = GROQ_SAMPLE_RATE, **kwargs, ): + """Initialize Groq TTS service. + + Args: + api_key: Groq API key for authentication. + output_format: Audio output format. Defaults to "wav". + params: Additional input parameters for voice customization. + model_name: TTS model to use. Defaults to "playai-tts". + voice_id: Voice identifier to use. Defaults to "Celeste-PlayAI". + sample_rate: Audio sample rate. Must be 48000 Hz for Groq TTS. + **kwargs: Additional arguments passed to parent TTSService class. + """ if sample_rate != self.GROQ_SAMPLE_RATE: logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ") @@ -71,10 +98,23 @@ class GroqTTSService(TTSService): self._client = AsyncGroq(api_key=self._api_key) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Groq TTS service supports metrics generation. + """ return True @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Groq's TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech data. + """ logger.debug(f"{self}: Generating TTS [{text}]") measuring_ttfb = True await self.start_ttfb_metrics() diff --git a/src/pipecat/services/image_service.py b/src/pipecat/services/image_service.py index 27084f1d1..243e917a6 100644 --- a/src/pipecat/services/image_service.py +++ b/src/pipecat/services/image_service.py @@ -24,12 +24,14 @@ class ImageGenService(AIService): Processes TextFrames by using their content as prompts for image generation. Subclasses must implement the run_image_gen method to provide actual image generation functionality using their specific AI service. - - Args: - **kwargs: Additional arguments passed to the parent AIService. """ def __init__(self, **kwargs): + """Initialize the image generation service. + + Args: + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) # Renders the image. Returns an Image object. diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index f7779df98..6ef85951a 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -128,11 +128,6 @@ class LLMService(AIService): parallel and sequential execution modes. Provides event handlers for completion timeouts and function call lifecycle events. - Args: - run_in_parallel: Whether to run function calls in parallel or sequentially. - Defaults to True. - **kwargs: Additional arguments passed to the parent AIService. - Event handlers: on_completion_timeout: Called when an LLM completion timeout occurs. on_function_calls_started: Called when function calls are received and @@ -155,6 +150,13 @@ class LLMService(AIService): adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter def __init__(self, run_in_parallel: bool = True, **kwargs): + """Initialize the LLM service. + + Args: + run_in_parallel: Whether to run function calls in parallel or sequentially. + Defaults to True. + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._run_in_parallel = run_in_parallel self._start_callbacks = {} diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index ca03f7a06..57a83e3e5 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""LMNT text-to-speech service implementation.""" + import json from typing import AsyncGenerator, Optional @@ -35,6 +37,14 @@ except ModuleNotFoundError as e: def language_to_lmnt_language(language: Language) -> Optional[str]: + """Convert a Language enum to LMNT language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding LMNT language code, or None if not supported. + """ BASE_LANGUAGES = { Language.DE: "de", Language.EN: "en", @@ -71,6 +81,13 @@ def language_to_lmnt_language(language: Language) -> Optional[str]: class LmntTTSService(InterruptibleTTSService): + """LMNT real-time text-to-speech service. + + Provides real-time text-to-speech synthesis using LMNT's WebSocket API. + Supports streaming audio generation with configurable voice models and + language settings. + """ + def __init__( self, *, @@ -81,6 +98,16 @@ class LmntTTSService(InterruptibleTTSService): model: str = "aurora", **kwargs, ): + """Initialize the LMNT TTS service. + + Args: + api_key: LMNT API key for authentication. + voice_id: ID of the voice to use for synthesis. + sample_rate: Audio sample rate. If None, uses default. + language: Language for synthesis. Defaults to English. + model: TTS model to use. Defaults to "aurora". + **kwargs: Additional arguments passed to parent InterruptibleTTSService. + """ super().__init__( push_stop_frames=True, pause_frame_processing=True, @@ -99,35 +126,71 @@ class LmntTTSService(InterruptibleTTSService): self._receive_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as LMNT service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to LMNT service language format. + + Args: + language: The language to convert. + + Returns: + The LMNT-specific language code, or None if not supported. + """ return language_to_lmnt_language(language) async def start(self, frame: StartFrame): + """Start the LMNT TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the LMNT TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the LMNT TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame downstream with special handling for stop conditions. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False async def _connect(self): + """Connect to LMNT WebSocket and start receive task.""" await self._connect_websocket() if self._websocket and not self._receive_task: self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): + """Disconnect from LMNT WebSocket and clean up tasks.""" if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None @@ -181,11 +244,13 @@ class LmntTTSService(InterruptibleTTSService): self._websocket = None def _get_websocket(self): + """Get the WebSocket connection if available.""" if self._websocket: return self._websocket raise Exception("Websocket not connected") async def flush_audio(self): + """Flush any pending audio synthesis.""" if not self._websocket or self._websocket.closed: return await self._get_websocket().send(json.dumps({"flush": True})) @@ -216,7 +281,14 @@ class LmntTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate TTS audio from text.""" + """Generate TTS audio from text using LMNT's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/mcp_service.py b/src/pipecat/services/mcp_service.py index 48202e8f9..31a29354b 100644 --- a/src/pipecat/services/mcp_service.py +++ b/src/pipecat/services/mcp_service.py @@ -35,10 +35,6 @@ class MCPClient(BaseObject): to LLMs. Supports both stdio and SSE server connections with automatic tool registration and schema conversion. - Args: - server_params: Server connection parameters (stdio or SSE). - **kwargs: Additional arguments passed to the parent BaseObject. - Raises: TypeError: If server_params is not a supported parameter type. """ @@ -48,6 +44,12 @@ class MCPClient(BaseObject): server_params: Tuple[StdioServerParameters, SseServerParameters, StreamableHttpParameters], **kwargs, ): + """Initialize the MCP client with server parameters. + + Args: + server_params: Server connection parameters (stdio or SSE). + **kwargs: Additional arguments passed to the parent BaseObject. + """ super().__init__(**kwargs) self._server_params = server_params self._session = ClientSession @@ -190,6 +192,7 @@ class MCPClient(BaseObject): async def _streamable_http_register_tools(self, llm) -> ToolsSchema: """Register all available mcp tools with the LLM service using streamable HTTP. + Args: llm: The Pipecat LLM service to register tools with Returns: diff --git a/src/pipecat/services/mem0/memory.py b/src/pipecat/services/mem0/memory.py index 8c93d72c9..32ea829c5 100644 --- a/src/pipecat/services/mem0/memory.py +++ b/src/pipecat/services/mem0/memory.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Mem0 memory service integration for Pipecat. + +This module provides a memory service that integrates with Mem0 to store +and retrieve conversational memories, enhancing LLM context with relevant +historical information. +""" + from typing import Any, Dict, List, Optional from loguru import logger @@ -31,14 +38,21 @@ class Mem0MemoryService(FrameProcessor): This service intercepts message frames in the pipeline, stores them in Mem0, and enhances context with relevant memories before passing them downstream. - - Args: - api_key (str): The API key for accessing Mem0's API - user_id (str): The user ID to associate with memories in Mem0 - params (InputParams, optional): Configuration parameters for memory retrieval + Supports both local and cloud-based Mem0 configurations. """ class InputParams(BaseModel): + """Configuration parameters for Mem0 memory service. + + Parameters: + search_limit: Maximum number of memories to retrieve per query. + search_threshold: Minimum similarity threshold for memory retrieval. + api_version: API version to use for Mem0 client operations. + system_prompt: Prefix text for memory context messages. + add_as_system_message: Whether to add memories as system messages. + position: Position to insert memory messages in context. + """ + search_limit: int = Field(default=10, ge=1) search_threshold: float = Field(default=0.1, ge=0.0, le=1.0) api_version: str = Field(default="v2") @@ -56,6 +70,19 @@ class Mem0MemoryService(FrameProcessor): run_id: Optional[str] = None, params: Optional[InputParams] = None, ): + """Initialize the Mem0 memory service. + + Args: + api_key: The API key for accessing Mem0's cloud API. + local_config: Local configuration for Mem0 client (alternative to cloud API). + user_id: The user ID to associate with memories in Mem0. + agent_id: The agent ID to associate with memories in Mem0. + run_id: The run ID to associate with memories in Mem0. + params: Configuration parameters for memory retrieval and storage. + + Raises: + ValueError: If none of user_id, agent_id, or run_id are provided. + """ # Important: Call the parent class __init__ first super().__init__() @@ -86,7 +113,7 @@ class Mem0MemoryService(FrameProcessor): """Store messages in Mem0. Args: - messages: List of message dictionaries to store + messages: List of message dictionaries to store in memory. """ try: logger.debug(f"Storing {len(messages)} messages in Mem0") @@ -110,10 +137,10 @@ class Mem0MemoryService(FrameProcessor): """Retrieve relevant memories from Mem0. Args: - query: The query to search for relevant memories + query: The query to search for relevant memories. Returns: - List of relevant memory dictionaries + List of relevant memory dictionaries matching the query. """ try: logger.debug(f"Retrieving memories for query: {query}") @@ -154,8 +181,8 @@ class Mem0MemoryService(FrameProcessor): """Enhance the LLM context with relevant memories. Args: - context: The OpenAILLMContext to enhance - query: The query to search for relevant memories + context: The OpenAILLMContext to enhance with memory information. + query: The query to search for relevant memories. """ # Skip if this is the same query we just processed if self.last_query == query: @@ -184,8 +211,8 @@ class Mem0MemoryService(FrameProcessor): """Process incoming frames, intercept context frames for memory integration. Args: - frame: The incoming frame to process - direction: The direction of frame flow in the pipeline + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. """ await super().process_frame(frame, direction) diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 86f954755..4faf88b1c 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""MiniMax text-to-speech service implementation. + +This module provides integration with MiniMax's T2A (Text-to-Audio) API +for streaming text-to-speech synthesis. +""" + import json from typing import AsyncGenerator, Optional @@ -25,6 +31,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts def language_to_minimax_language(language: Language) -> Optional[str]: + """Convert a Language enum to MiniMax language format. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding MiniMax language name, or None if not supported. + """ BASE_LANGUAGES = { Language.AR: "Arabic", Language.CS: "Czech", @@ -71,24 +85,18 @@ def language_to_minimax_language(language: Language) -> Optional[str]: class MiniMaxHttpTTSService(TTSService): """Text-to-speech service using MiniMax's T2A (Text-to-Audio) API. + Provides streaming text-to-speech synthesis using MiniMax's HTTP API + with support for various voice settings, emotions, and audio configurations. + Supports real-time audio streaming with configurable voice parameters. + Platform documentation: https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643 - - Args: - api_key: MiniMax API key for authentication. - group_id: MiniMax Group ID to identify project. - model: TTS model name (default: "speech-02-turbo"). Options include - "speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo". - voice_id: Voice identifier (default: "Calm_Woman"). - aiohttp_session: aiohttp.ClientSession for API communication. - sample_rate: Output audio sample rate in Hz (default: None, set from pipeline). - params: Additional configuration parameters. """ class InputParams(BaseModel): """Configuration parameters for MiniMax TTS. - Attributes: + Parameters: language: Language for TTS generation. speed: Speech speed (range: 0.5 to 2.0). volume: Speech volume (range: 0 to 10). @@ -117,6 +125,19 @@ class MiniMaxHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the MiniMax TTS service. + + Args: + api_key: MiniMax API key for authentication. + group_id: MiniMax Group ID to identify project. + model: TTS model name. Defaults to "speech-02-turbo". Options include + "speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo". + voice_id: Voice identifier. Defaults to "Calm_Woman". + aiohttp_session: aiohttp.ClientSession for API communication. + sample_rate: Output audio sample rate in Hz. If None, uses pipeline default. + params: Additional configuration parameters. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or MiniMaxHttpTTSService.InputParams() @@ -175,28 +196,62 @@ class MiniMaxHttpTTSService(TTSService): self._settings["english_normalization"] = params.english_normalization def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as MiniMax service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to MiniMax service language format. + + Args: + language: The language to convert. + + Returns: + The MiniMax-specific language name, or None if not supported. + """ return language_to_minimax_language(language) def set_model_name(self, model: str): - """Set the TTS model to use""" + """Set the TTS model to use. + + Args: + model: The model name to use for synthesis. + """ self._model_name = model def set_voice(self, voice: str): - """Set the voice to use""" + """Set the voice to use. + + Args: + voice: The voice identifier to use for synthesis. + """ self._voice_id = voice if "voice_setting" in self._settings: self._settings["voice_setting"]["voice_id"] = voice async def start(self, frame: StartFrame): + """Start the MiniMax TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["audio_setting"]["sample_rate"] = self.sample_rate logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}") @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using MiniMax's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") headers = { diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index 6fe44a057..88be8a2a1 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Moondream vision service implementation. + +This module provides integration with the Moondream vision-language model +for image analysis and description generation. +""" + import asyncio from typing import AsyncGenerator @@ -23,7 +29,15 @@ except ModuleNotFoundError as e: def detect_device(): - """Detects the appropriate device to run on, and return the device and dtype.""" + """Detect the appropriate device to run on. + + Detects available hardware acceleration and selects the best device + and data type for optimal performance. + + Returns: + tuple: A tuple containing (device, dtype) where device is a torch.device + and dtype is the recommended torch data type for that device. + """ try: import intel_extension_for_pytorch @@ -40,9 +54,24 @@ def detect_device(): class MoondreamService(VisionService): + """Moondream vision-language model service. + + Provides image analysis and description generation using the Moondream + vision-language model. Supports various hardware acceleration options + including CUDA, MPS, and Intel XPU. + """ + def __init__( self, *, model="vikhyatk/moondream2", revision="2024-08-26", use_cpu=False, **kwargs ): + """Initialize the Moondream service. + + Args: + model: Hugging Face model identifier for the Moondream model. + revision: Specific model revision to use. + use_cpu: Whether to force CPU usage instead of hardware acceleration. + **kwargs: Additional arguments passed to the parent VisionService. + """ super().__init__(**kwargs) self.set_model_name(model) @@ -65,6 +94,15 @@ class MoondreamService(VisionService): logger.debug("Loaded Moondream model") async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: + """Analyze an image and generate a description. + + Args: + frame: Vision frame containing the image data and optional question text. + + Yields: + Frame: TextFrame containing the generated image description, or ErrorFrame + if analysis fails. + """ if not self._model: logger.error(f"{self} error: Moondream model not available ({self.model_name})") yield ErrorFrame("Moondream model not available") @@ -73,6 +111,14 @@ class MoondreamService(VisionService): logger.debug(f"Analyzing image: {frame}") def get_image_description(frame: VisionImageRawFrame): + """Generate description for the given image frame. + + Args: + frame: Vision frame containing image data and question. + + Returns: + str: Generated description of the image. + """ image = Image.frombytes(frame.format, frame.size, frame.image) image_embeds = self._model.encode_image(image) description = self._model.answer_question( diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index b92e5966c..8f795f36c 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Neuphonic text-to-speech service implementations. + +This module provides WebSocket and HTTP-based integrations with Neuphonic's +text-to-speech API for real-time audio synthesis. +""" + import asyncio import base64 import json @@ -42,6 +48,14 @@ except ModuleNotFoundError as e: def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: + """Convert a Language enum to Neuphonic language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Neuphonic language code, or None if not supported. + """ BASE_LANGUAGES = { Language.DE: "de", Language.EN: "en", @@ -69,7 +83,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: class NeuphonicTTSService(InterruptibleTTSService): + """Neuphonic real-time text-to-speech service using WebSocket streaming. + + Provides real-time text-to-speech synthesis using Neuphonic's WebSocket API. + Supports interruption handling, keepalive connections, and configurable voice + parameters for high-quality speech generation. + """ + class InputParams(BaseModel): + """Input parameters for Neuphonic TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -84,6 +112,17 @@ class NeuphonicTTSService(InterruptibleTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Neuphonic TTS service. + + Args: + api_key: Neuphonic API key for authentication. + voice_id: ID of the voice to use for synthesis. + url: WebSocket URL for the Neuphonic API. + sample_rate: Audio sample rate in Hz. Defaults to 22050. + encoding: Audio encoding format. Defaults to "pcm_linear". + params: Additional input parameters for TTS configuration. + **kwargs: Additional arguments passed to parent InterruptibleTTSService. + """ super().__init__( aggregate_sentences=True, push_text_frames=False, @@ -114,12 +153,26 @@ class NeuphonicTTSService(InterruptibleTTSService): self._keepalive_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Neuphonic service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Neuphonic service language format. + + Args: + language: The language to convert. + + Returns: + The Neuphonic-specific language code, or None if not supported. + """ return language_to_neuphonic_lang_code(language) async def _update_settings(self, settings: Mapping[str, Any]): + """Update service settings and reconnect with new configuration.""" if "voice_id" in settings: self.set_voice(settings["voice_id"]) @@ -129,28 +182,56 @@ class NeuphonicTTSService(InterruptibleTTSService): logger.info(f"Switching TTS to settings: [{self._settings}]") async def start(self, frame: StartFrame): + """Start the Neuphonic TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the Neuphonic TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the Neuphonic TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def flush_audio(self): + """Flush any pending audio synthesis by sending stop command.""" if self._websocket: msg = {"text": ""} await self._websocket.send(json.dumps(msg)) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame downstream with special handling for stop conditions. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with special handling for speech control. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) # If we received a TTSSpeakFrame and the LLM response included text (it @@ -164,6 +245,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self.resume_processing_frames() async def _connect(self): + """Connect to Neuphonic WebSocket and start background tasks.""" await self._connect_websocket() if self._websocket and not self._receive_task: @@ -173,6 +255,7 @@ class NeuphonicTTSService(InterruptibleTTSService): self._keepalive_task = self.create_task(self._keepalive_task_handler()) async def _disconnect(self): + """Disconnect from Neuphonic WebSocket and clean up tasks.""" if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None @@ -184,6 +267,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._disconnect_websocket() async def _connect_websocket(self): + """Establish WebSocket connection to Neuphonic API.""" try: if self._websocket and self._websocket.open: return @@ -209,6 +293,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): + """Close WebSocket connection and clean up state.""" try: await self.stop_all_metrics() @@ -222,6 +307,7 @@ class NeuphonicTTSService(InterruptibleTTSService): self._websocket = None async def _receive_messages(self): + """Receive and process messages from Neuphonic WebSocket.""" async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager): if isinstance(message, str): msg = json.loads(message) @@ -233,6 +319,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self.push_frame(frame) async def _keepalive_task_handler(self): + """Handle keepalive messages to maintain WebSocket connection.""" KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3 while True: self.reset_watchdog() @@ -240,6 +327,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._send_text("") async def _send_text(self, text: str): + """Send text to Neuphonic WebSocket for synthesis.""" if self._websocket: msg = {"text": text} logger.debug(f"Sending text to websocket: {msg}") @@ -247,6 +335,14 @@ class NeuphonicTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Neuphonic's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"Generating TTS: [{text}]") try: @@ -274,19 +370,21 @@ class NeuphonicTTSService(InterruptibleTTSService): class NeuphonicHttpTTSService(TTSService): - """Neuphonic Text-to-Speech service using HTTP streaming. + """Neuphonic text-to-speech service using HTTP streaming. - Args: - api_key: Neuphonic API key - voice_id: ID of the voice to use - url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com") - sample_rate: Sample rate for audio output (default: 22050Hz) - encoding: Audio encoding format (default: "pcm_linear") - params: Additional parameters for TTS generation including language and speed - **kwargs: Additional keyword arguments passed to the parent class + Provides text-to-speech synthesis using Neuphonic's HTTP API with server-sent + events for streaming audio delivery. Suitable for applications that prefer + HTTP-based communication over WebSocket connections. """ class InputParams(BaseModel): + """Input parameters for Neuphonic HTTP TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -301,6 +399,17 @@ class NeuphonicHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Neuphonic HTTP TTS service. + + Args: + api_key: Neuphonic API key for authentication. + voice_id: ID of the voice to use for synthesis. + url: Base URL for the Neuphonic HTTP API. + sample_rate: Audio sample rate in Hz. Defaults to 22050. + encoding: Audio encoding format. Defaults to "pcm_linear". + params: Additional input parameters for TTS configuration. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or NeuphonicHttpTTSService.InputParams() @@ -316,12 +425,38 @@ class NeuphonicHttpTTSService(TTSService): self.set_voice(voice_id) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Neuphonic HTTP service supports metrics generation. + """ return True + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Neuphonic service language format. + + Args: + language: The language to convert. + + Returns: + The Neuphonic-specific language code, or None if not supported. + """ + return language_to_neuphonic_lang_code(language) + async def start(self, frame: StartFrame): + """Start the Neuphonic HTTP TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) async def flush_audio(self): + """Flush any pending audio synthesis. + + Note: + HTTP-based service doesn't require explicit flushing. + """ pass @traced_tts @@ -329,9 +464,10 @@ class NeuphonicHttpTTSService(TTSService): """Generate speech from text using Neuphonic streaming API. Args: - text: The text to convert to speech + text: The text to convert to speech. + Yields: - Frames containing audio data and status information + Frame: Audio frames containing the synthesized speech and status information. """ logger.debug(f"Generating TTS: [{text}]") diff --git a/src/pipecat/services/nim/llm.py b/src/pipecat/services/nim/llm.py index a637a6602..052b94274 100644 --- a/src/pipecat/services/nim/llm.py +++ b/src/pipecat/services/nim/llm.py @@ -21,12 +21,6 @@ class NimLLMService(OpenAILLMService): This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining compatibility with the OpenAI-style interface. It specifically handles the difference in token usage reporting between NIM (incremental) and OpenAI (final summary). - - Args: - api_key: The API key for accessing NVIDIA's NIM API. - base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1". - model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -37,6 +31,14 @@ class NimLLMService(OpenAILLMService): model: str = "nvidia/llama-3.1-nemotron-70b-instruct", **kwargs, ): + """Initialize the NimLLMService. + + Args: + api_key: The API key for accessing NVIDIA's NIM API. + base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1". + model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) # Counters for accumulating token usage metrics self._prompt_tokens = 0 diff --git a/src/pipecat/services/ollama/llm.py b/src/pipecat/services/ollama/llm.py index 9fc5ab840..c84505704 100644 --- a/src/pipecat/services/ollama/llm.py +++ b/src/pipecat/services/ollama/llm.py @@ -14,12 +14,14 @@ class OLLamaLLMService(OpenAILLMService): This service extends OpenAILLMService to work with locally hosted OLLama models, providing a compatible interface for running large language models locally. - - Args: - model: The OLLama model to use. Defaults to "llama2". - base_url: The base URL for the OLLama API endpoint. - Defaults to "http://localhost:11434/v1". """ def __init__(self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1"): + """Initialize OLLama LLM service. + + Args: + model: The OLLama model to use. Defaults to "llama2". + base_url: The base URL for the OLLama API endpoint. + Defaults to "http://localhost:11434/v1". + """ super().__init__(model=model, base_url=base_url, api_key="ollama") diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index b307b3e21..f88134fa2 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -48,16 +48,6 @@ class BaseOpenAILLMService(LLMService): to an OpenAILLMContext object. The context defines what is sent to the LLM for completion, including user, assistant, and system messages, as well as tool choices and function call configurations. - - Args: - model: The OpenAI model name to use (e.g., "gpt-4.1", "gpt-4o"). - api_key: OpenAI API key. If None, uses environment variable. - base_url: Custom base URL for OpenAI API. If None, uses default. - organization: OpenAI organization ID. - project: OpenAI project ID. - default_headers: Additional HTTP headers to include in requests. - params: Input parameters for model configuration and behavior. - **kwargs: Additional arguments passed to the parent LLMService. """ class InputParams(BaseModel): @@ -103,6 +93,18 @@ class BaseOpenAILLMService(LLMService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the BaseOpenAILLMService. + + Args: + model: The OpenAI model name to use (e.g., "gpt-4.1", "gpt-4o"). + api_key: OpenAI API key. If None, uses environment variable. + base_url: Custom base URL for OpenAI API. If None, uses default. + organization: OpenAI organization ID. + project: OpenAI project ID. + default_headers: Additional HTTP headers to include in requests. + params: Input parameters for model configuration and behavior. + **kwargs: Additional arguments passed to the parent LLMService. + """ super().__init__(**kwargs) params = params or BaseOpenAILLMService.InputParams() diff --git a/src/pipecat/services/openai/image.py b/src/pipecat/services/openai/image.py index 3d7f6cb70..76ec14902 100644 --- a/src/pipecat/services/openai/image.py +++ b/src/pipecat/services/openai/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI image generation service implementation. + +This module provides integration with OpenAI's DALL-E image generation API +for creating images from text prompts. +""" + import io from typing import AsyncGenerator, Literal, Optional @@ -21,6 +27,13 @@ from pipecat.services.image_service import ImageGenService class OpenAIImageGenService(ImageGenService): + """OpenAI DALL-E image generation service. + + Provides image generation capabilities using OpenAI's DALL-E models. + Supports various image sizes and can generate images from text prompts + with configurable quality and style parameters. + """ + def __init__( self, *, @@ -30,6 +43,15 @@ class OpenAIImageGenService(ImageGenService): image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"], model: str = "dall-e-3", ): + """Initialize the OpenAI image generation service. + + Args: + api_key: OpenAI API key for authentication. + base_url: Custom base URL for OpenAI API. If None, uses default. + aiohttp_session: HTTP session for downloading generated images. + image_size: Target size for generated images. + model: DALL-E model to use for generation. Defaults to "dall-e-3". + """ super().__init__() self.set_model_name(model) self._image_size = image_size @@ -37,6 +59,14 @@ class OpenAIImageGenService(ImageGenService): self._aiohttp_session = aiohttp_session async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate an image from a text prompt using OpenAI's DALL-E. + + Args: + prompt: Text description of the image to generate. + + Yields: + Frame: URLImageRawFrame containing the generated image data. + """ logger.debug(f"Generating image from prompt: {prompt}") image = await self._client.images.generate( diff --git a/src/pipecat/services/openai/llm.py b/src/pipecat/services/openai/llm.py index c27eab867..7919dd159 100644 --- a/src/pipecat/services/openai/llm.py +++ b/src/pipecat/services/openai/llm.py @@ -61,11 +61,6 @@ class OpenAILLMService(BaseOpenAILLMService): Provides a complete OpenAI LLM service with context aggregation support. Uses the BaseOpenAILLMService for core functionality and adds OpenAI-specific context aggregator creation. - - Args: - model: The OpenAI model name to use. Defaults to "gpt-4.1". - params: Input parameters for model configuration. - **kwargs: Additional arguments passed to the parent BaseOpenAILLMService. """ def __init__( @@ -75,6 +70,13 @@ class OpenAILLMService(BaseOpenAILLMService): params: Optional[BaseOpenAILLMService.InputParams] = None, **kwargs, ): + """Initialize OpenAI LLM service. + + Args: + model: The OpenAI model name to use. Defaults to "gpt-4.1". + params: Input parameters for model configuration. + **kwargs: Additional arguments passed to the parent BaseOpenAILLMService. + """ super().__init__(model=model, params=params, **kwargs) def create_context_aggregator( diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index 173205aa0..208c68d34 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI Speech-to-Text service implementation using OpenAI's transcription API.""" + from typing import Optional from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -15,15 +17,6 @@ class OpenAISTTService(BaseWhisperSTTService): Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key set via the api_key parameter or OPENAI_API_KEY environment variable. - - Args: - model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe". - api_key: OpenAI API key. Defaults to None. - base_url: API base URL. Defaults to None. - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to BaseWhisperSTTService. """ def __init__( @@ -37,6 +30,17 @@ class OpenAISTTService(BaseWhisperSTTService): temperature: Optional[float] = None, **kwargs, ): + """Initialize OpenAI STT service. + + Args: + model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe". + api_key: OpenAI API key. Defaults to None. + base_url: API base URL. Defaults to None. + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to BaseWhisperSTTService. + """ super().__init__( model=model, api_key=api_key, diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index 946d5e396..6e82496a4 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI text-to-speech service implementation. + +This module provides integration with OpenAI's text-to-speech API for +generating high-quality synthetic speech from text input. +""" + from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger @@ -43,16 +49,8 @@ class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz. - - Args: - api_key: OpenAI API key. Defaults to None. - voice: Voice ID to use. Defaults to "alloy". - model: TTS model to use. Defaults to "gpt-4o-mini-tts". - sample_rate: Output audio sample rate in Hz. Defaults to None. - **kwargs: Additional keyword arguments passed to TTSService. - - The service returns PCM-encoded audio at the specified sample rate. - + Supports multiple voice models and configurable parameters for high-quality + speech synthesis with streaming audio output. """ OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz @@ -68,6 +66,17 @@ class OpenAITTSService(TTSService): instructions: Optional[str] = None, **kwargs, ): + """Initialize OpenAI TTS service. + + Args: + api_key: OpenAI API key for authentication. If None, uses environment variable. + base_url: Custom base URL for OpenAI API. If None, uses default. + voice: Voice ID to use for synthesis. Defaults to "alloy". + model: TTS model to use. Defaults to "gpt-4o-mini-tts". + sample_rate: Output audio sample rate in Hz. If None, uses OpenAI's default 24kHz. + instructions: Optional instructions to guide voice synthesis behavior. + **kwargs: Additional keyword arguments passed to TTSService. + """ if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE: logger.warning( f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. " @@ -81,13 +90,28 @@ class OpenAITTSService(TTSService): self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as OpenAI TTS service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the TTS model to use. + + Args: + model: The model name to use for text-to-speech synthesis. + """ logger.info(f"Switching TTS model to: [{model}]") self.set_model_name(model) async def start(self, frame: StartFrame): + """Start the OpenAI TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) if self.sample_rate != self.OPENAI_SAMPLE_RATE: logger.warning( @@ -97,6 +121,14 @@ class OpenAITTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using OpenAI's TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech data. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/openai_realtime_beta/azure.py b/src/pipecat/services/openai_realtime_beta/azure.py index a6cde33f9..cfa279840 100644 --- a/src/pipecat/services/openai_realtime_beta/azure.py +++ b/src/pipecat/services/openai_realtime_beta/azure.py @@ -26,12 +26,6 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, using Azure's authentication headers and endpoint format. Provides the same real-time audio and text communication capabilities as the base OpenAI service. - - Args: - api_key: The API key for the Azure OpenAI service. - base_url: The full Azure WebSocket endpoint URL including api-version and deployment. - Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" - **kwargs: Additional arguments passed to parent OpenAIRealtimeBetaLLMService. """ def __init__( @@ -41,6 +35,14 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): base_url: str, **kwargs, ): + """Initialize Azure Realtime Beta LLM service. + + Args: + api_key: The API key for the Azure OpenAI service. + base_url: The full Azure WebSocket endpoint URL including api-version and deployment. + Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + **kwargs: Additional arguments passed to parent OpenAIRealtimeBetaLLMService. + """ super().__init__(base_url=base_url, api_key=api_key, **kwargs) self.api_key = api_key self.base_url = base_url diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 7caee0ece..cb1c0a9f5 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -37,14 +37,16 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext): Extends the standard OpenAI LLM context to support real-time session properties, instruction management, and conversion between standard message formats and realtime conversation items. - - Args: - messages: Initial conversation messages. Defaults to None. - tools: Available function tools. Defaults to None. - **kwargs: Additional arguments passed to parent OpenAILLMContext. """ def __init__(self, messages=None, tools=None, **kwargs): + """Initialize the OpenAIRealtimeLLMContext. + + Args: + messages: Initial conversation messages. Defaults to None. + tools: Available function tools. Defaults to None. + **kwargs: Additional arguments passed to parent OpenAILLMContext. + """ super().__init__(messages=messages, tools=tools, **kwargs) self.__setup_local() diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 695cd3015..6a45add17 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -18,13 +18,7 @@ from pydantic import BaseModel, ConfigDict, Field class InputAudioTranscription(BaseModel): - """Configuration for audio transcription settings. - - Parameters: - model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). - language: Optional language code for transcription. - prompt: Optional transcription hint text. - """ + """Configuration for audio transcription settings.""" model: str = "gpt-4o-transcribe" language: Optional[str] @@ -36,6 +30,13 @@ class InputAudioTranscription(BaseModel): language: Optional[str] = None, prompt: Optional[str] = None, ): + """Initialize InputAudioTranscription. + + Args: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ super().__init__(model=model, language=language, prompt=prompt) @@ -881,6 +882,8 @@ class TokenDetails(BaseModel): audio_tokens: Optional[int] = 0 class Config: + """Pydantic configuration for TokenDetails.""" + extra = "allow" diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 286fb41a1..ce21e33ad 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -96,17 +96,6 @@ class OpenAIRealtimeBetaLLMService(LLMService): Implements the OpenAI Realtime API Beta with WebSocket communication for low-latency bidirectional audio and text interactions. Supports function calling, conversation management, and real-time transcription. - - Args: - api_key: OpenAI API key for authentication. - model: OpenAI model name. Defaults to "gpt-4o-realtime-preview-2025-06-03". - base_url: WebSocket base URL for the realtime API. - Defaults to "wss://api.openai.com/v1/realtime". - session_properties: Configuration properties for the realtime session. - If None, uses default SessionProperties. - start_audio_paused: Whether to start with audio input paused. Defaults to False. - send_transcription_frames: Whether to emit transcription frames. Defaults to True. - **kwargs: Additional arguments passed to parent LLMService. """ # Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one. @@ -123,6 +112,19 @@ class OpenAIRealtimeBetaLLMService(LLMService): send_transcription_frames: bool = True, **kwargs, ): + """Initialize the OpenAI Realtime Beta LLM service. + + Args: + api_key: OpenAI API key for authentication. + model: OpenAI model name. Defaults to "gpt-4o-realtime-preview-2025-06-03". + base_url: WebSocket base URL for the realtime API. + Defaults to "wss://api.openai.com/v1/realtime". + session_properties: Configuration properties for the realtime session. + If None, uses default SessionProperties. + start_audio_paused: Whether to start with audio input paused. Defaults to False. + send_transcription_frames: Whether to emit transcription frames. Defaults to True. + **kwargs: Additional arguments passed to parent LLMService. + """ full_url = f"{base_url}?model={model}" super().__init__(base_url=full_url, **kwargs) diff --git a/src/pipecat/services/openpipe/llm.py b/src/pipecat/services/openpipe/llm.py index 59dd543de..25257e294 100644 --- a/src/pipecat/services/openpipe/llm.py +++ b/src/pipecat/services/openpipe/llm.py @@ -33,15 +33,6 @@ class OpenPipeLLMService(OpenAILLMService): Extends OpenAI's LLM service to integrate with OpenPipe's fine-tuning and monitoring platform. Provides enhanced request logging and tagging capabilities for model training and evaluation. - - Args: - model: The model name to use. Defaults to "gpt-4.1". - api_key: OpenAI API key for authentication. If None, reads from environment. - base_url: Custom OpenAI API endpoint URL. Uses default if None. - openpipe_api_key: OpenPipe API key for enhanced features. If None, reads from environment. - openpipe_base_url: OpenPipe API endpoint URL. Defaults to "https://app.openpipe.ai/api/v1". - tags: Optional dictionary of tags to apply to all requests for tracking. - **kwargs: Additional arguments passed to parent OpenAILLMService. """ def __init__( @@ -55,6 +46,17 @@ class OpenPipeLLMService(OpenAILLMService): tags: Optional[Dict[str, str]] = None, **kwargs, ): + """Initialize OpenPipe LLM service. + + Args: + model: The model name to use. Defaults to "gpt-4.1". + api_key: OpenAI API key for authentication. If None, reads from environment. + base_url: Custom OpenAI API endpoint URL. Uses default if None. + openpipe_api_key: OpenPipe API key for enhanced features. If None, reads from environment. + openpipe_base_url: OpenPipe API endpoint URL. Defaults to "https://app.openpipe.ai/api/v1". + tags: Optional dictionary of tags to apply to all requests for tracking. + **kwargs: Additional arguments passed to parent OpenAILLMService. + """ super().__init__( model=model, api_key=api_key, diff --git a/src/pipecat/services/openrouter/llm.py b/src/pipecat/services/openrouter/llm.py index 85d1662fe..97a9d336a 100644 --- a/src/pipecat/services/openrouter/llm.py +++ b/src/pipecat/services/openrouter/llm.py @@ -22,13 +22,6 @@ class OpenRouterLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to OpenRouter's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing OpenRouter's API. If None, will attempt - to read from environment variables. - model: The model identifier to use. Defaults to "openai/gpt-4o-2024-11-20". - base_url: The base URL for OpenRouter API. Defaults to "https://openrouter.ai/api/v1". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -39,6 +32,15 @@ class OpenRouterLLMService(OpenAILLMService): base_url: str = "https://openrouter.ai/api/v1", **kwargs, ): + """Initialize the OpenRouter LLM service. + + Args: + api_key: The API key for accessing OpenRouter's API. If None, will attempt + to read from environment variables. + model: The model identifier to use. Defaults to "openai/gpt-4o-2024-11-20". + base_url: The base URL for OpenRouter API. Defaults to "https://openrouter.ai/api/v1". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__( api_key=api_key, base_url=base_url, diff --git a/src/pipecat/services/perplexity/llm.py b/src/pipecat/services/perplexity/llm.py index 049181cc9..ae80b3942 100644 --- a/src/pipecat/services/perplexity/llm.py +++ b/src/pipecat/services/perplexity/llm.py @@ -27,12 +27,6 @@ class PerplexityLLMService(OpenAILLMService): This service extends OpenAILLMService to work with Perplexity's API while maintaining compatibility with the OpenAI-style interface. It specifically handles the difference in token usage reporting between Perplexity (incremental) and OpenAI (final summary). - - Args: - api_key: The API key for accessing Perplexity's API. - base_url: The base URL for Perplexity's API. Defaults to "https://api.perplexity.ai". - model: The model identifier to use. Defaults to "sonar". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -43,6 +37,14 @@ class PerplexityLLMService(OpenAILLMService): model: str = "sonar", **kwargs, ): + """Initialize the Perplexity LLM service. + + Args: + api_key: The API key for accessing Perplexity's API. + base_url: The base URL for Perplexity's API. Defaults to "https://api.perplexity.ai". + model: The model identifier to use. Defaults to "sonar". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) # Counters for accumulating token usage metrics self._prompt_tokens = 0 diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index 65caa3650..90176dab8 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Piper TTS service implementation.""" + from typing import AsyncGenerator, Optional import aiohttp @@ -24,12 +26,9 @@ from pipecat.utils.tracing.service_decorators import traced_tts class PiperTTSService(TTSService): """Piper TTS service implementation. - Provides integration with Piper's TTS server. - - Args: - base_url: API base URL - aiohttp_session: aiohttp ClientSession - sample_rate: Output sample rate + Provides integration with Piper's HTTP TTS server for text-to-speech + synthesis. Supports streaming audio generation with configurable sample + rates and automatic WAV header removal. """ def __init__( @@ -42,6 +41,14 @@ class PiperTTSService(TTSService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the Piper TTS service. + + Args: + base_url: Base URL for the Piper TTS HTTP server. + aiohttp_session: aiohttp ClientSession for making HTTP requests. + sample_rate: Output sample rate. If None, uses the voice model's native rate. + **kwargs: Additional arguments passed to the parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) if base_url.endswith("/"): @@ -53,17 +60,22 @@ class PiperTTSService(TTSService): self._settings = {"base_url": base_url} def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Piper service supports metrics generation. + """ return True @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Piper API. + """Generate speech from text using Piper's HTTP API. Args: - text: The text to convert to speech + text: The text to convert to speech. Yields: - Frames containing audio data and status information + Frame: Audio frames containing the synthesized speech and status frames. """ logger.debug(f"{self}: Generating TTS [{text}]") headers = { diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 34fc6c81a..65c9fd41e 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""PlayHT text-to-speech service implementations. + +This module provides integration with PlayHT's text-to-speech API +supporting both WebSocket streaming and HTTP-based synthesis. +""" + import io import json import struct @@ -42,6 +48,14 @@ except ModuleNotFoundError as e: def language_to_playht_language(language: Language) -> Optional[str]: + """Convert a Language enum to PlayHT language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding PlayHT language code, or None if not supported. + """ BASE_LANGUAGES = { Language.AF: "afrikans", Language.AM: "amharic", @@ -96,7 +110,22 @@ def language_to_playht_language(language: Language) -> Optional[str]: class PlayHTTTSService(InterruptibleTTSService): + """PlayHT WebSocket-based text-to-speech service. + + Provides real-time text-to-speech synthesis using PlayHT's WebSocket API. + Supports streaming audio generation with configurable voice engines and + language settings. + """ + class InputParams(BaseModel): + """Input parameters for PlayHT TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + seed: Random seed for voice consistency. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 seed: Optional[int] = None @@ -113,6 +142,18 @@ class PlayHTTTSService(InterruptibleTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the PlayHT WebSocket TTS service. + + Args: + api_key: PlayHT API key for authentication. + user_id: PlayHT user ID for authentication. + voice_url: URL of the voice to use for synthesis. + voice_engine: Voice engine to use. Defaults to "Play3.0-mini". + sample_rate: Audio sample rate. If None, uses default. + output_format: Audio output format. Defaults to "wav". + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to parent InterruptibleTTSService. + """ super().__init__( pause_frame_processing=True, sample_rate=sample_rate, @@ -140,30 +181,60 @@ class PlayHTTTSService(InterruptibleTTSService): self.set_voice(voice_url) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as PlayHT service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to PlayHT service language format. + + Args: + language: The language to convert. + + Returns: + The PlayHT-specific language code, or None if not supported. + """ return language_to_playht_language(language) async def start(self, frame: StartFrame): + """Start the PlayHT TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the PlayHT TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the PlayHT TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def _connect(self): + """Connect to PlayHT WebSocket and start receive task.""" await self._connect_websocket() if self._websocket and not self._receive_task: self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): + """Disconnect from PlayHT WebSocket and clean up tasks.""" if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None @@ -171,6 +242,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self._disconnect_websocket() async def _connect_websocket(self): + """Connect to PlayHT websocket.""" try: if self._websocket and self._websocket.open: return @@ -194,6 +266,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): + """Disconnect from PlayHT websocket.""" try: await self.stop_all_metrics() @@ -207,6 +280,7 @@ class PlayHTTTSService(InterruptibleTTSService): self._websocket = None async def _get_websocket_url(self): + """Retrieve WebSocket URL from PlayHT API.""" async with aiohttp.ClientSession() as session: async with session.post( "https://api.play.ht/api/v4/websocket-auth", @@ -235,16 +309,19 @@ class PlayHTTTSService(InterruptibleTTSService): raise Exception(f"Failed to get WebSocket URL: {response.status}") def _get_websocket(self): + """Get the WebSocket connection if available.""" if self._websocket: return self._websocket raise Exception("Websocket not connected") async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + """Handle interruption by stopping metrics and clearing request ID.""" await super()._handle_interruption(frame, direction) await self.stop_all_metrics() self._request_id = None async def _receive_messages(self): + """Receive messages from PlayHT websocket.""" async for message in self._get_websocket(): if isinstance(message, bytes): # Skip the WAV header message @@ -273,6 +350,14 @@ class PlayHTTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using PlayHT's WebSocket API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -316,7 +401,22 @@ class PlayHTTTSService(InterruptibleTTSService): class PlayHTHttpTTSService(TTSService): + """PlayHT HTTP-based text-to-speech service. + + Provides text-to-speech synthesis using PlayHT's HTTP API for simpler, + non-streaming synthesis. Suitable for use cases where streaming is not + required and simpler integration is preferred. + """ + class InputParams(BaseModel): + """Input parameters for PlayHT HTTP TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + seed: Random seed for voice consistency. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 seed: Optional[int] = None @@ -333,6 +433,18 @@ class PlayHTHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the PlayHT HTTP TTS service. + + Args: + api_key: PlayHT API key for authentication. + user_id: PlayHT user ID for authentication. + voice_url: URL of the voice to use for synthesis. + voice_engine: Voice engine to use. Defaults to "Play3.0-mini". + protocol: Protocol to use ("http" or "ws"). Defaults to "http". + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or PlayHTHttpTTSService.InputParams() @@ -369,10 +481,16 @@ class PlayHTHttpTTSService(TTSService): self.set_voice(voice_url) async def start(self, frame: StartFrame): + """Start the PlayHT HTTP TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["sample_rate"] = self.sample_rate def _create_options(self) -> TTSOptions: + """Create TTSOptions object from current settings.""" language_str = self._settings["language"] playht_language = None if language_str: @@ -392,13 +510,34 @@ class PlayHTHttpTTSService(TTSService): ) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as PlayHT HTTP service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to PlayHT service language format. + + Args: + language: The language to convert. + + Returns: + The PlayHT-specific language code, or None if not supported. + """ return language_to_playht_language(language) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using PlayHT's HTTP API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/qwen/llm.py b/src/pipecat/services/qwen/llm.py index 2ffc6bc80..648cbd9e8 100644 --- a/src/pipecat/services/qwen/llm.py +++ b/src/pipecat/services/qwen/llm.py @@ -16,12 +16,6 @@ class QwenLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Qwen's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing Qwen's API (DashScope API key). - base_url: Base URL for Qwen API. Defaults to "https://dashscope-intl.aliyuncs.com/compatible-mode/v1". - model: The model identifier to use. Defaults to "qwen-plus". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -32,6 +26,14 @@ class QwenLLMService(OpenAILLMService): model: str = "qwen-plus", **kwargs, ): + """Initialize the Qwen LLM service. + + Args: + api_key: The API key for accessing Qwen's API (DashScope API key). + base_url: Base URL for Qwen API. Defaults to "https://dashscope-intl.aliyuncs.com/compatible-mode/v1". + model: The model identifier to use. Defaults to "qwen-plus". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) logger.info(f"Initialized Qwen LLM service with model: {model}") diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 821eafb23..663bda28f 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Rime text-to-speech service implementations. + +This module provides both WebSocket and HTTP-based text-to-speech services +using Rime's API for streaming and batch audio synthesis. +""" + import base64 import json import uuid @@ -47,7 +53,7 @@ def language_to_rime_language(language: Language) -> str: language: The pipecat Language enum value. Returns: - str: Three-letter language code used by Rime (e.g., 'eng' for English). + Three-letter language code used by Rime (e.g., 'eng' for English). """ LANGUAGE_MAP = { Language.DE: "ger", @@ -67,7 +73,15 @@ class RimeTTSService(AudioContextWordTTSService): """ class InputParams(BaseModel): - """Configuration parameters for Rime TTS service.""" + """Configuration parameters for Rime TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + speed_alpha: Speech speed multiplier. Defaults to 1.0. + reduce_latency: Whether to reduce latency at potential quality cost. + pause_between_brackets: Whether to add pauses between bracketed content. + phonemize_between_brackets: Whether to phonemize bracketed content. + """ language: Optional[Language] = Language.EN speed_alpha: Optional[float] = 1.0 @@ -96,6 +110,8 @@ class RimeTTSService(AudioContextWordTTSService): model: Model ID to use for synthesis. sample_rate: Audio sample rate in Hz. params: Additional configuration parameters. + text_aggregator: Custom text aggregator for processing input text. + **kwargs: Additional arguments passed to parent class. """ # Initialize with parent class settings for proper frame handling super().__init__( @@ -135,14 +151,30 @@ class RimeTTSService(AudioContextWordTTSService): self._cumulative_time = 0 # Accumulates time across messages def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Rime service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> str | None: - """Convert pipecat language to Rime language code.""" + """Convert pipecat language to Rime language code. + + Args: + language: The language to convert. + + Returns: + The Rime-specific language code, or None if not supported. + """ return language_to_rime_language(language) async def set_model(self, model: str): - """Update the TTS model.""" + """Update the TTS model. + + Args: + model: The model name to use for synthesis. + """ self._model = model await super().set_model(model) @@ -159,18 +191,30 @@ class RimeTTSService(AudioContextWordTTSService): return {"operation": "eos"} async def start(self, frame: StartFrame): - """Start the service and establish websocket connection.""" + """Start the service and establish websocket connection. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["samplingRate"] = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): - """Stop the service and close connection.""" + """Stop the service and close connection. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): - """Cancel current operation and clean up.""" + """Cancel current operation and clean up. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() @@ -261,6 +305,7 @@ class RimeTTSService(AudioContextWordTTSService): return word_pairs async def flush_audio(self): + """Flush any pending audio synthesis.""" if not self._context_id or not self._websocket: return @@ -310,7 +355,12 @@ class RimeTTSService(AudioContextWordTTSService): self._context_id = None async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): - """Push frame and handle end-of-turn conditions.""" + """Push frame and handle end-of-turn conditions. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): if isinstance(frame, TTSStoppedFrame): @@ -318,13 +368,13 @@ class RimeTTSService(AudioContextWordTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text. + """Generate speech from text using Rime's streaming API. Args: text: The text to convert to speech. Yields: - Frames containing audio data and timing information. + Frame: Audio frames containing the synthesized speech. """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -354,7 +404,24 @@ class RimeTTSService(AudioContextWordTTSService): class RimeHttpTTSService(TTSService): + """Rime HTTP-based text-to-speech service. + + Provides text-to-speech synthesis using Rime's HTTP API for batch processing. + Suitable for use cases where streaming is not required. + """ + class InputParams(BaseModel): + """Configuration parameters for Rime HTTP TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + pause_between_brackets: Whether to add pauses between bracketed content. + phonemize_between_brackets: Whether to phonemize bracketed content. + inline_speed_alpha: Inline speed control markup. + speed_alpha: Speech speed multiplier. Defaults to 1.0. + reduce_latency: Whether to reduce latency at potential quality cost. + """ + language: Optional[Language] = Language.EN pause_between_brackets: Optional[bool] = False phonemize_between_brackets: Optional[bool] = False @@ -373,6 +440,17 @@ class RimeHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize Rime HTTP TTS service. + + Args: + api_key: Rime API key for authentication. + voice_id: ID of the voice to use. + aiohttp_session: Shared aiohttp session for HTTP requests. + model: Model ID to use for synthesis. + sample_rate: Audio sample rate in Hz. + params: Additional configuration parameters. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RimeHttpTTSService.InputParams() @@ -396,14 +474,34 @@ class RimeHttpTTSService(TTSService): self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Rime HTTP service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> str | None: - """Convert pipecat language to Rime language code.""" + """Convert pipecat language to Rime language code. + + Args: + language: The language to convert. + + Returns: + The Rime-specific language code, or None if not supported. + """ return language_to_rime_language(language) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Rime's HTTP API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") headers = { diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py index 0d2330bef..ba8750f91 100644 --- a/src/pipecat/services/riva/stt.py +++ b/src/pipecat/services/riva/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""NVIDIA Riva Speech-to-Text service implementations for real-time and batch transcription.""" + import asyncio from typing import AsyncGenerator, List, Mapping, Optional @@ -87,7 +89,20 @@ def language_to_riva_language(language: Language) -> Optional[str]: class RivaSTTService(STTService): + """Real-time speech-to-text service using NVIDIA Riva streaming ASR. + + Provides real-time transcription capabilities using NVIDIA's Riva ASR models + through streaming recognition. Supports interim results and continuous audio + processing for low-latency applications. + """ + class InputParams(BaseModel): + """Configuration parameters for Riva STT service. + + Parameters: + language: Target language for transcription. Defaults to EN_US. + """ + language: Optional[Language] = Language.EN_US def __init__( @@ -103,6 +118,16 @@ class RivaSTTService(STTService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Riva STT service. + + Args: + api_key: NVIDIA API key for authentication. + server: Riva server address. Defaults to NVIDIA Cloud Function endpoint. + model_function_map: Mapping containing 'function_id' and 'model_name' for the ASR model. + sample_rate: Audio sample rate in Hz. If None, uses pipeline default. + params: Additional configuration parameters for Riva. + **kwargs: Additional arguments passed to STTService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RivaSTTService.InputParams() @@ -148,9 +173,23 @@ class RivaSTTService(STTService): self._response_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + False - this service does not support metrics generation. + """ return False async def set_model(self, model: str): + """Set the ASR model for transcription. + + Args: + model: Model name to set. + + Note: + Model cannot be changed after initialization. Use model_function_map + parameter in constructor instead. + """ logger.warning(f"Cannot set model after initialization. Set model and function id like so:") example = {"function_id": "", "model_name": ""} logger.warning( @@ -158,6 +197,11 @@ class RivaSTTService(STTService): ) async def start(self, frame: StartFrame): + """Start the Riva STT service and initialize streaming configuration. + + Args: + frame: StartFrame indicating pipeline start. + """ await super().start(frame) if self._config: @@ -203,10 +247,20 @@ class RivaSTTService(STTService): self._response_task = self.create_task(self._response_task_handler()) async def stop(self, frame: EndFrame): + """Stop the Riva STT service and clean up resources. + + Args: + frame: EndFrame indicating pipeline stop. + """ await super().stop(frame) await self._stop_tasks() async def cancel(self, frame: CancelFrame): + """Cancel the Riva STT service operation. + + Args: + frame: CancelFrame indicating operation cancellation. + """ await super().cancel(frame) await self._stop_tasks() @@ -289,18 +343,39 @@ class RivaSTTService(STTService): self._response_queue.task_done() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None - transcription results are pushed to the pipeline via frames. + """ await self.start_ttfb_metrics() await self.start_processing_metrics() await self._queue.put(audio) yield None def __next__(self) -> bytes: + """Get the next audio chunk for Riva processing. + + Returns: + Audio bytes from the queue. + + Raises: + StopIteration: When the thread is no longer running. + """ if not self._thread_running: raise StopIteration future = asyncio.run_coroutine_threadsafe(self._queue.get(), self.get_event_loop()) return future.result() def __iter__(self): + """Return iterator for audio chunk processing. + + Returns: + Self as iterator. + """ return self @@ -310,17 +385,20 @@ class RivaSegmentedSTTService(SegmentedSTTService): By default, his service uses NVIDIA's Riva Canary ASR API to perform speech-to-text transcription on audio segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. - - Args: - api_key: NVIDIA API key for authentication - server: Riva server address (defaults to NVIDIA Cloud Function endpoint) - model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID - sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate - params: Additional configuration parameters for Riva - **kwargs: Additional arguments passed to SegmentedSTTService """ class InputParams(BaseModel): + """Configuration parameters for Riva segmented STT service. + + Parameters: + language: Target language for transcription. Defaults to EN_US. + profanity_filter: Whether to filter profanity from results. + automatic_punctuation: Whether to add automatic punctuation. + verbatim_transcripts: Whether to return verbatim transcripts. + boosted_lm_words: List of words to boost in language model. + boosted_lm_score: Score boost for specified words. + """ + language: Optional[Language] = Language.EN_US profanity_filter: bool = False automatic_punctuation: bool = True @@ -341,6 +419,16 @@ class RivaSegmentedSTTService(SegmentedSTTService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Riva segmented STT service. + + Args: + api_key: NVIDIA API key for authentication + server: Riva server address (defaults to NVIDIA Cloud Function endpoint) + model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate + params: Additional configuration parameters for Riva + **kwargs: Additional arguments passed to SegmentedSTTService + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RivaSegmentedSTTService.InputParams() @@ -380,7 +468,14 @@ class RivaSegmentedSTTService(SegmentedSTTService): self._settings = {"language": self._language_enum} def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert pipecat Language enum to Riva's language code.""" + """Convert pipecat Language enum to Riva's language code. + + Args: + language: Language enum value. + + Returns: + Riva language code or None if not supported. + """ return language_to_riva_language(language) def _initialize_client(self): @@ -435,10 +530,23 @@ class RivaSegmentedSTTService(SegmentedSTTService): return config def can_generate_metrics(self) -> bool: - """Indicates whether this service can generate processing metrics.""" + """Check if this service can generate processing metrics. + + Returns: + True - this service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the ASR model for transcription. + + Args: + model: Model name to set. + + Note: + Model cannot be changed after initialization. Use model_function_map + parameter in constructor instead. + """ logger.warning(f"Cannot set model after initialization. Set model and function id like so:") example = {"function_id": "", "model_name": ""} logger.warning( @@ -446,13 +554,21 @@ class RivaSegmentedSTTService(SegmentedSTTService): ) async def start(self, frame: StartFrame): - """Initialize the service when the pipeline starts.""" + """Initialize the service when the pipeline starts. + + Args: + frame: StartFrame indicating pipeline start. + """ await super().start(frame) self._initialize_client() self._config = self._create_recognition_config() async def set_language(self, language: Language): - """Set the language for the STT service.""" + """Set the language for the STT service. + + Args: + language: Target language for transcription. + """ logger.info(f"Switching STT language to: [{language}]") self._language_enum = language self._language = self.language_to_service_language(language) or "en-US" @@ -539,7 +655,11 @@ class RivaSegmentedSTTService(SegmentedSTTService): class ParakeetSTTService(RivaSTTService): - """Deprecated: Use RivaSTTService instead.""" + """Deprecated speech-to-text service using NVIDIA Parakeet models. + + This class is deprecated. Use RivaSTTService instead for equivalent functionality + with Parakeet models by specifying the appropriate model_function_map. + """ def __init__( self, @@ -554,6 +674,16 @@ class ParakeetSTTService(RivaSTTService): params: Optional[RivaSTTService.InputParams] = None, # Use parent class's type **kwargs, ): + """Initialize the Parakeet STT service. + + Args: + api_key: NVIDIA API key for authentication. + server: Riva server address. Defaults to NVIDIA Cloud Function endpoint. + model_function_map: Mapping containing 'function_id' and 'model_name' for Parakeet model. + sample_rate: Audio sample rate in Hz. If None, uses pipeline default. + params: Additional configuration parameters for Riva. + **kwargs: Additional arguments passed to RivaSTTService. + """ super().__init__( api_key=api_key, server=server, diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 31850ea17..b75f09db0 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""NVIDIA Riva text-to-speech service implementation. + +This module provides integration with NVIDIA Riva's TTS services through +gRPC API for high-quality speech synthesis. +""" + import asyncio import os from typing import AsyncGenerator, Mapping, Optional @@ -37,7 +43,21 @@ RIVA_TTS_TIMEOUT_SECS = 5 class RivaTTSService(TTSService): + """NVIDIA Riva text-to-speech service. + + Provides high-quality text-to-speech synthesis using NVIDIA Riva's + cloud-based TTS models. Supports multiple voices, languages, and + configurable quality settings. + """ + class InputParams(BaseModel): + """Input parameters for Riva TTS configuration. + + Parameters: + language: Language code for synthesis. Defaults to US English. + quality: Audio quality setting (0-100). Defaults to 20. + """ + language: Optional[Language] = Language.EN_US quality: Optional[int] = 20 @@ -55,6 +75,17 @@ class RivaTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the NVIDIA Riva TTS service. + + Args: + api_key: NVIDIA API key for authentication. + server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint. + voice_id: Voice model identifier. Defaults to multilingual Ray voice. + sample_rate: Audio sample rate. If None, uses service default. + model_function_map: Dictionary containing function_id and model_name for the TTS model. + params: Additional configuration parameters for TTS synthesis. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RivaTTSService.InputParams() @@ -82,6 +113,13 @@ class RivaTTSService(TTSService): ) async def set_model(self, model: str): + """Attempt to set the TTS model. + + Note: Model cannot be changed after initialization for Riva service. + + Args: + model: The model name to set (operation not supported). + """ logger.warning(f"Cannot set model after initialization. Set model and function id like so:") example = {"function_id": "", "model_name": ""} logger.warning( @@ -90,6 +128,15 @@ class RivaTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using NVIDIA Riva TTS. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech data. + """ + def read_audio_responses(queue: asyncio.Queue): def add_response(r): asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop()) @@ -139,6 +186,12 @@ class RivaTTSService(TTSService): class FastPitchTTSService(RivaTTSService): + """Deprecated FastPitch TTS service. + + This class is deprecated. Use RivaTTSService instead for new implementations. + Provides backward compatibility for existing FastPitch TTS integrations. + """ + def __init__( self, *, @@ -153,6 +206,17 @@ class FastPitchTTSService(RivaTTSService): params: Optional[RivaTTSService.InputParams] = None, **kwargs, ): + """Initialize the deprecated FastPitch TTS service. + + Args: + api_key: NVIDIA API key for authentication. + server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint. + voice_id: Voice model identifier. Defaults to Female-1 voice. + sample_rate: Audio sample rate. If None, uses service default. + model_function_map: Dictionary containing function_id and model_name for FastPitch model. + params: Additional configuration parameters for TTS synthesis. + **kwargs: Additional arguments passed to parent RivaTTSService. + """ super().__init__( api_key=api_key, server=server, diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index e0ddfb9bc..c11489e66 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -29,12 +29,6 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore This service extends OpenAILLMService to connect to SambaNova's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing SambaNova API. - model: The model identifier to use. Defaults to "Llama-4-Maverick-17B-128E-Instruct". - base_url: The base URL for SambaNova API. Defaults to "https://api.sambanova.ai/v1". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -45,6 +39,14 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore base_url: str = "https://api.sambanova.ai/v1", **kwargs: Dict[Any, Any], ) -> None: + """Initialize SambaNova LLM service. + + Args: + api_key: The API key for accessing SambaNova API. + model: The model identifier to use. Defaults to "Llama-4-Maverick-17B-128E-Instruct". + base_url: The base URL for SambaNova API. Defaults to "https://api.sambanova.ai/v1". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client( diff --git a/src/pipecat/services/sambanova/stt.py b/src/pipecat/services/sambanova/stt.py index ed518d6b8..71a709420 100644 --- a/src/pipecat/services/sambanova/stt.py +++ b/src/pipecat/services/sambanova/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""SambaNova's Speech-to-Text service implementation for real-time transcription.""" + from typing import Any, Optional from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -12,16 +14,9 @@ from pipecat.transcriptions.language import Language class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore """SambaNova Whisper speech-to-text service. + Uses SambaNova's Whisper API to convert audio to text. Requires a SambaNova API key set via the api_key parameter or SAMBANOVA_API_KEY environment variable. - Args: - model: Whisper model to use. Defaults to "Whisper-Large-v3". - api_key: SambaNova API key. Defaults to None. - base_url: API base URL. Defaults to "https://api.sambanova.ai/v1". - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to `pipecat.services.whisper.base_stt.BaseWhisperSTTService`. """ def __init__( @@ -35,6 +30,17 @@ class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore temperature: Optional[float] = None, **kwargs: Any, ) -> None: + """Initialize SambaNova STT service. + + Args: + model: Whisper model to use. Defaults to "Whisper-Large-v3". + api_key: SambaNova API key. Defaults to None. + base_url: API base URL. Defaults to "https://api.sambanova.ai/v1". + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to `pipecat.services.whisper.base_stt.BaseWhisperSTTService`. + """ super().__init__( model=model, api_key=api_key, diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index f9ce4e70f..eee4048cb 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Sarvam AI text-to-speech service implementation.""" + import base64 from typing import AsyncGenerator, Optional @@ -25,7 +27,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts def language_to_sarvam_language(language: Language) -> Optional[str]: - """Convert Pipecat Language enum to Sarvam AI language codes.""" + """Convert Pipecat Language enum to Sarvam AI language codes. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Sarvam AI language code, or None if not supported. + """ LANGUAGE_MAP = { Language.BN: "bn-IN", # Bengali Language.EN: "en-IN", # English (India) @@ -50,15 +59,6 @@ class SarvamTTSService(TTSService): Indian languages. Provides control over voice characteristics like pitch, pace, and loudness. - Args: - api_key: Sarvam AI API subscription key. - voice_id: Speaker voice ID (e.g., "anushka", "meera"). - model: TTS model to use ("bulbul:v1" or "bulbul:v2"). - aiohttp_session: Shared aiohttp session for making requests. - base_url: Sarvam AI API base URL. - sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000). - params: Additional voice and preprocessing parameters. - Example: ```python tts = SarvamTTSService( @@ -76,6 +76,16 @@ class SarvamTTSService(TTSService): """ class InputParams(BaseModel): + """Input parameters for Sarvam TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English (India). + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + pace: Speech pace multiplier (0.3 to 3.0). Defaults to 1.0. + loudness: Volume multiplier (0.1 to 3.0). Defaults to 1.0. + enable_preprocessing: Whether to enable text preprocessing. Defaults to False. + """ + language: Optional[Language] = Language.EN pitch: Optional[float] = Field(default=0.0, ge=-0.75, le=0.75) pace: Optional[float] = Field(default=1.0, ge=0.3, le=3.0) @@ -94,6 +104,18 @@ class SarvamTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Sarvam TTS service. + + Args: + api_key: Sarvam AI API subscription key. + voice_id: Speaker voice ID (e.g., "anushka", "meera"). Defaults to "anushka". + model: TTS model to use ("bulbul:v1" or "bulbul:v2"). Defaults to "bulbul:v2". + aiohttp_session: Shared aiohttp session for making requests. + base_url: Sarvam AI API base URL. Defaults to "https://api.sarvam.ai". + sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000). If None, uses default. + params: Additional voice and preprocessing parameters. If None, uses defaults. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or SarvamTTSService.InputParams() @@ -116,17 +138,43 @@ class SarvamTTSService(TTSService): self.set_voice(voice_id) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Sarvam service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Sarvam AI language format. + + Args: + language: The language to convert. + + Returns: + The Sarvam AI-specific language code, or None if not supported. + """ return language_to_sarvam_language(language) async def start(self, frame: StartFrame): + """Start the Sarvam TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["sample_rate"] = self.sample_rate @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Sarvam AI's API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index 5ddcbcba8..6ba7da4b8 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Simli video service for real-time avatar generation.""" + import asyncio import numpy as np @@ -31,12 +33,26 @@ except ModuleNotFoundError as e: class SimliVideoService(FrameProcessor): + """Simli video service for real-time avatar generation. + + Provides real-time avatar video generation by processing audio frames + and producing synchronized video output using the Simli API. Handles + audio resampling, video frame processing, and connection management. + """ + def __init__( self, simli_config: SimliConfig, use_turn_server: bool = False, latency_interval: int = 0, ): + """Initialize the Simli video service. + + Args: + simli_config: Configuration object for Simli client settings. + use_turn_server: Whether to use TURN server for connection. Defaults to False. + latency_interval: Latency interval setting for video processing. Defaults to 0. + """ super().__init__() self._simli_client = SimliClient(simli_config, use_turn_server, latency_interval) @@ -49,6 +65,7 @@ class SimliVideoService(FrameProcessor): self._video_task: asyncio.Task = None async def _start_connection(self): + """Start the connection to Simli service and begin processing tasks.""" if not self._initialized: await self._simli_client.Initialize() self._initialized = True @@ -61,6 +78,7 @@ class SimliVideoService(FrameProcessor): self._video_task = self.create_task(self._consume_and_process_video()) async def _consume_and_process_audio(self): + """Consume audio frames from Simli and push them downstream.""" await self._pipecat_resampler_event.wait() audio_iterator = self._simli_client.getAudioStreamIterator() async for audio_frame in WatchdogAsyncIterator(audio_iterator, manager=self.task_manager): @@ -78,6 +96,7 @@ class SimliVideoService(FrameProcessor): ) async def _consume_and_process_video(self): + """Consume video frames from Simli and convert them to output frames.""" await self._pipecat_resampler_event.wait() video_iterator = self._simli_client.getVideoStreamIterator(targetFormat="rgb24") async for video_frame in WatchdogAsyncIterator(video_iterator, manager=self.task_manager): @@ -91,6 +110,12 @@ class SimliVideoService(FrameProcessor): await self.push_frame(convertedFrame) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle Simli video generation. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): await self.push_frame(frame, direction) @@ -127,6 +152,7 @@ class SimliVideoService(FrameProcessor): await self.push_frame(frame, direction) async def _stop(self): + """Stop the Simli client and cancel processing tasks.""" await self._simli_client.stop() if self._audio_task: await self.cancel_task(self._audio_task) diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index e659b403b..db777c77f 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -33,13 +33,6 @@ class STTService(AIService): Provides common functionality for STT services including audio passthrough, muting, settings management, and audio processing. Subclasses must implement the run_stt method to provide actual speech recognition. - - Args: - audio_passthrough: Whether to pass audio frames downstream after processing. - Defaults to True. - sample_rate: The sample rate for audio input. If None, will be determined - from the start frame. - **kwargs: Additional arguments passed to the parent AIService. """ def __init__( @@ -49,6 +42,15 @@ class STTService(AIService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the STT service. + + Args: + audio_passthrough: Whether to pass audio frames downstream after processing. + Defaults to True. + sample_rate: The sample rate for audio input. If None, will be determined + from the start frame. + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._audio_passthrough = audio_passthrough self._init_sample_rate = sample_rate @@ -173,14 +175,16 @@ class SegmentedSTTService(STTService): Requires VAD to be enabled in the pipeline to function properly. Maintains a small audio buffer to account for the delay between actual speech start and VAD detection. - - Args: - sample_rate: The sample rate for audio input. If None, will be determined - from the start frame. - **kwargs: Additional arguments passed to the parent STTService. """ def __init__(self, *, sample_rate: Optional[int] = None, **kwargs): + """Initialize the segmented STT service. + + Args: + sample_rate: The sample rate for audio input. If None, will be determined + from the start frame. + **kwargs: Additional arguments passed to the parent STTService. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._content = None self._wave = None diff --git a/src/pipecat/services/tavus/video.py b/src/pipecat/services/tavus/video.py index e97da71b9..999d712d0 100644 --- a/src/pipecat/services/tavus/video.py +++ b/src/pipecat/services/tavus/video.py @@ -4,7 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""This module implements Tavus as a sink transport layer""" +"""Tavus video service implementation for avatar-based video generation. + +This module implements Tavus as a sink transport layer, providing video +avatar functionality through Tavus's streaming API. +""" import asyncio from typing import Optional @@ -31,23 +35,15 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue class TavusVideoService(AIService): - """ - Service class that proxies audio to Tavus and receives both audio and video in return. + """Service that proxies audio to Tavus and receives audio and video in return. - It uses the `TavusTransportClient` to manage the session and handle communication. When - audio is sent, Tavus responds with both audio and video streams, which are then routed - through Pipecat’s media pipeline. + Uses the TavusTransportClient to manage sessions and handle communication. + When audio is sent, Tavus responds with both audio and video streams, which + are routed through Pipecat's media pipeline. - In use cases such as with `DailyTransport`, this results in two distinct virtual rooms: - - **Tavus room**: Contains the Tavus Avatar and the Pipecat Bot. - - **User room**: Contains the Pipecat Bot and the user. - - Args: - api_key (str): Tavus API key used for authentication. - replica_id (str): ID of the Tavus voice replica to use for speech synthesis. - persona_id (str): ID of the Tavus persona. Defaults to "pipecat-stream" to use the Pipecat TTS voice. - session (aiohttp.ClientSession): Async HTTP session used for communication with Tavus. - **kwargs: Additional arguments passed to the parent `AIService` class. + In use cases with DailyTransport, this creates two distinct virtual rooms: + - Tavus room: Contains the Tavus Avatar and the Pipecat Bot + - User room: Contains the Pipecat Bot and the user """ def __init__( @@ -59,6 +55,15 @@ class TavusVideoService(AIService): session: aiohttp.ClientSession, **kwargs, ) -> None: + """Initialize the Tavus video service. + + Args: + api_key: Tavus API key used for authentication. + replica_id: ID of the Tavus voice replica to use for speech synthesis. + persona_id: ID of the Tavus persona. Defaults to "pipecat-stream" for Pipecat TTS voice. + session: Async HTTP session used for communication with Tavus. + **kwargs: Additional arguments passed to the parent AIService class. + """ super().__init__(**kwargs) self._api_key = api_key self._session = session @@ -77,6 +82,11 @@ class TavusVideoService(AIService): self._transport_destination: Optional[str] = "stream" async def setup(self, setup: FrameProcessorSetup): + """Set up the Tavus video service. + + Args: + setup: Frame processor setup configuration. + """ await super().setup(setup) callbacks = TavusCallbacks( on_participant_joined=self._on_participant_joined, @@ -99,15 +109,18 @@ class TavusVideoService(AIService): await self._client.setup(setup) async def cleanup(self): + """Clean up the service and release resources.""" await super().cleanup() await self._client.cleanup() self._client = None async def _on_participant_left(self, participant, reason): + """Handle participant leaving the session.""" participant_id = participant["id"] logger.info(f"Participant left {participant_id}, reason: {reason}") async def _on_participant_joined(self, participant): + """Handle participant joining the session.""" participant_id = participant["id"] logger.info(f"Participant joined {participant_id}") if not self._other_participant_has_joined: @@ -124,6 +137,7 @@ class TavusVideoService(AIService): async def _on_participant_video_frame( self, participant_id: str, video_frame: VideoFrame, video_source: str ): + """Handle incoming video frames from participants.""" frame = OutputImageRawFrame( image=video_frame.buffer, size=(video_frame.width, video_frame.height), @@ -135,6 +149,7 @@ class TavusVideoService(AIService): async def _on_participant_audio_data( self, participant_id: str, audio: AudioData, audio_source: str ): + """Handle incoming audio data from participants.""" frame = OutputAudioRawFrame( audio=audio.audio_frames, sample_rate=audio.sample_rate, @@ -144,12 +159,27 @@ class TavusVideoService(AIService): await self.push_frame(frame) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Tavus service supports metrics generation. + """ return True async def get_persona_name(self) -> str: + """Get the name of the current persona. + + Returns: + The persona name from the Tavus client. + """ return await self._client.get_persona_name() async def start(self, frame: StartFrame): + """Start the Tavus video service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._client.start(frame) if self._transport_destination: @@ -157,16 +187,32 @@ class TavusVideoService(AIService): await self._create_send_task() async def stop(self, frame: EndFrame): + """Stop the Tavus video service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._end_conversation() await self._cancel_send_task() async def cancel(self, frame: CancelFrame): + """Cancel the Tavus video service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._end_conversation() await self._cancel_send_task() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames through the service. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): @@ -178,25 +224,30 @@ class TavusVideoService(AIService): await self.push_frame(frame, direction) async def _handle_interruptions(self): + """Handle interruption events by resetting send tasks and notifying client.""" await self._cancel_send_task() await self._create_send_task() await self._client.send_interrupt_message() async def _end_conversation(self): + """End the current conversation and reset state.""" await self._client.stop() self._other_participant_has_joined = False async def _create_send_task(self): + """Create the audio sending task if it doesn't exist.""" if not self._send_task: self._queue = WatchdogQueue(self.task_manager) self._send_task = self.create_task(self._send_task_handler()) async def _cancel_send_task(self): + """Cancel the audio sending task if it exists.""" if self._send_task: await self.cancel_task(self._send_task) self._send_task = None async def _handle_audio_frame(self, frame: OutputAudioRawFrame): + """Process audio frames for sending to Tavus.""" sample_rate = self._client.out_sample_rate # 40 ms of audio chunk_size = int((sample_rate * 2) / 25) @@ -215,6 +266,7 @@ class TavusVideoService(AIService): self._audio_buffer = self._audio_buffer[chunk_size:] async def _send_task_handler(self): + """Handle sending audio frames to the Tavus client.""" while True: frame = await self._queue.get() if isinstance(frame, OutputAudioRawFrame) and self._client: diff --git a/src/pipecat/services/together/llm.py b/src/pipecat/services/together/llm.py index e445be676..7a22c885a 100644 --- a/src/pipecat/services/together/llm.py +++ b/src/pipecat/services/together/llm.py @@ -16,12 +16,6 @@ class TogetherLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Together.ai's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key: The API key for accessing Together.ai's API. - base_url: The base URL for Together.ai API. Defaults to "https://api.together.xyz/v1". - model: The model identifier to use. Defaults to "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -32,6 +26,14 @@ class TogetherLLMService(OpenAILLMService): model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", **kwargs, ): + """Initialize Together.ai LLM service. + + Args: + api_key: The API key for accessing Together.ai's API. + base_url: The base URL for Together.ai API. Defaults to "https://api.together.xyz/v1". + model: The model identifier to use. Defaults to "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index e50244986..183ef1f19 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -50,21 +50,6 @@ class TTSService(AIService): Provides common functionality for TTS services including text aggregation, filtering, audio generation, and frame management. Supports configurable sentence aggregation, silence insertion, and frame processing control. - - Args: - aggregate_sentences: Whether to aggregate text into sentences before synthesis. - push_text_frames: Whether to push TextFrames and LLMFullResponseEndFrames. - push_stop_frames: Whether to automatically push TTSStoppedFrames. - stop_frame_timeout_s: Idle time before pushing TTSStoppedFrame when push_stop_frames is True. - push_silence_after_stop: Whether to push silence audio after TTSStoppedFrame. - silence_time_s: Duration of silence to push when push_silence_after_stop is True. - pause_frame_processing: Whether to pause frame processing during audio generation. - sample_rate: Output sample rate for generated audio. - text_aggregator: Custom text aggregator for processing incoming text. - text_filters: Sequence of text filters to apply after aggregation. - text_filter: Single text filter (deprecated, use text_filters). - transport_destination: Destination for generated audio frames. - **kwargs: Additional arguments passed to the parent AIService. """ def __init__( @@ -95,6 +80,23 @@ class TTSService(AIService): transport_destination: Optional[str] = None, **kwargs, ): + """Initialize the TTS service. + + Args: + aggregate_sentences: Whether to aggregate text into sentences before synthesis. + push_text_frames: Whether to push TextFrames and LLMFullResponseEndFrames. + push_stop_frames: Whether to automatically push TTSStoppedFrames. + stop_frame_timeout_s: Idle time before pushing TTSStoppedFrame when push_stop_frames is True. + push_silence_after_stop: Whether to push silence audio after TTSStoppedFrame. + silence_time_s: Duration of silence to push when push_silence_after_stop is True. + pause_frame_processing: Whether to pause frame processing during audio generation. + sample_rate: Output sample rate for generated audio. + text_aggregator: Custom text aggregator for processing incoming text. + text_filters: Sequence of text filters to apply after aggregation. + text_filter: Single text filter (deprecated, use text_filters). + transport_destination: Destination for generated audio frames. + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._aggregate_sentences: bool = aggregate_sentences self._push_text_frames: bool = push_text_frames @@ -428,12 +430,14 @@ class WordTTSService(TTSService): Word timestamps are useful to synchronize audio with text of the spoken words. This way only the spoken words are added to the conversation context. - - Args: - **kwargs: Additional arguments passed to the parent TTSService. """ def __init__(self, **kwargs): + """Initialize the Word TTS service. + + Args: + **kwargs: Additional arguments passed to the parent TTSService. + """ super().__init__(**kwargs) self._initial_word_timestamp = -1 self._words_task = None @@ -542,10 +546,6 @@ class WebsocketTTSService(TTSService, WebsocketService): Combines TTS functionality with websocket connectivity, providing automatic error handling and reconnection capabilities. - Args: - reconnect_on_error: Whether to automatically reconnect on websocket errors. - **kwargs: Additional arguments passed to parent classes. - Event handlers: on_connection_error: Called when a websocket connection error occurs. @@ -558,6 +558,12 @@ class WebsocketTTSService(TTSService, WebsocketService): """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): + """Initialize the Websocket TTS service. + + Args: + reconnect_on_error: Whether to automatically reconnect on websocket errors. + **kwargs: Additional arguments passed to parent classes. + """ TTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) self._register_event_handler("on_connection_error") @@ -572,12 +578,14 @@ class InterruptibleTTSService(WebsocketTTSService): Designed for TTS services that don't support word timestamps. Handles interruptions by reconnecting the websocket when the bot is speaking and gets interrupted. - - Args: - **kwargs: Additional arguments passed to the parent WebsocketTTSService. """ def __init__(self, **kwargs): + """Initialize the Interruptible TTS service. + + Args: + **kwargs: Additional arguments passed to the parent WebsocketTTSService. + """ super().__init__(**kwargs) # Indicates if the bot is speaking. If the bot is not speaking we don't @@ -611,10 +619,6 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): Combines word timestamp functionality with websocket connectivity. - Args: - reconnect_on_error: Whether to automatically reconnect on websocket errors. - **kwargs: Additional arguments passed to parent classes. - Event handlers: on_connection_error: Called when a websocket connection error occurs. @@ -627,6 +631,12 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): + """Initialize the Websocket Word TTS service. + + Args: + reconnect_on_error: Whether to automatically reconnect on websocket errors. + **kwargs: Additional arguments passed to parent classes. + """ WordTTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) self._register_event_handler("on_connection_error") @@ -641,12 +651,14 @@ class InterruptibleWordTTSService(WebsocketWordTTSService): For TTS services that support word timestamps but can't correlate generated audio with requested text. Handles interruptions by reconnecting when needed. - - Args: - **kwargs: Additional arguments passed to the parent WebsocketWordTTSService. """ def __init__(self, **kwargs): + """Initialize the Interruptible Word TTS service. + + Args: + **kwargs: Additional arguments passed to the parent WebsocketWordTTSService. + """ super().__init__(**kwargs) # Indicates if the bot is speaking. If the bot is not speaking we don't @@ -689,12 +701,14 @@ class AudioContextWordTTSService(WebsocketWordTTSService): The audio received from the TTS will be played in context order. That is, if we requested audio for a context "A" and then audio for context "B", the audio from context ID "A" will be played first. - - Args: - **kwargs: Additional arguments passed to the parent WebsocketWordTTSService. """ def __init__(self, **kwargs): + """Initialize the Audio Context Word TTS service. + + Args: + **kwargs: Additional arguments passed to the parent WebsocketWordTTSService. + """ super().__init__(**kwargs) self._contexts: Dict[str, asyncio.Queue] = {} self._audio_context_task = None diff --git a/src/pipecat/services/ultravox/stt.py b/src/pipecat/services/ultravox/stt.py index 8021f3c25..987593f02 100644 --- a/src/pipecat/services/ultravox/stt.py +++ b/src/pipecat/services/ultravox/stt.py @@ -44,13 +44,12 @@ except ModuleNotFoundError as e: class AudioBuffer: """Buffer to collect audio frames before processing. - Attributes: - frames: List of AudioRawFrames to process - started_at: Timestamp when speech started - is_processing: Flag to prevent concurrent processing + Manages the collection and state of audio frames during speech + recording sessions, including timing and processing flags. """ def __init__(self): + """Initialize the audio buffer.""" self.frames: List[AudioRawFrame] = [] self.started_at: Optional[float] = None self.is_processing: bool = False @@ -59,19 +58,17 @@ class AudioBuffer: class UltravoxModel: """Model wrapper for the Ultravox multimodal model. - This class handles loading and running the Ultravox model for speech-to-text. - - Args: - model_name: The name or path of the Ultravox model to load - - Attributes: - model_name: The name of the loaded model - engine: The vLLM engine for model inference - tokenizer: The tokenizer for the model - stop_token_ids: Optional token IDs to stop generation + This class handles loading and running the Ultravox model for speech-to-text + transcription using vLLM for efficient inference. """ def __init__(self, model_name: str = "fixie-ai/ultravox-v0_5-llama-3_1-8b"): + """Initialize the Ultravox model. + + Args: + model_name: The name or path of the Ultravox model to load. + Defaults to "fixie-ai/ultravox-v0_5-llama-3_1-8b". + """ self.model_name = model_name self._initialize_engine() self._initialize_tokenizer() @@ -95,10 +92,10 @@ class UltravoxModel: """Format chat messages into a prompt for the model. Args: - messages: List of message dictionaries with 'role' and 'content' + messages: List of message dictionaries with 'role' and 'content'. Returns: - str: Formatted prompt string + str: Formatted prompt string ready for model input. """ return self.tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True @@ -114,13 +111,13 @@ class UltravoxModel: """Generate text from audio input using the model. Args: - messages: List of message dictionaries - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - audio: Audio data as numpy array + messages: List of message dictionaries for conversation context. + temperature: Sampling temperature for generation randomness. + max_tokens: Maximum number of tokens to generate. + audio: Audio data as numpy array in float32 format. Yields: - str: JSON chunks of the generated response + str: JSON chunks of the generated response in OpenAI format. """ sampling_params = SamplingParams( temperature=temperature, max_tokens=max_tokens, stop_token_ids=self.stop_token_ids @@ -173,22 +170,9 @@ class UltravoxModel: class UltravoxSTTService(AIService): """Service to transcribe audio using the Ultravox multimodal model. - This service collects audio frames and processes them with Ultravox - to generate text transcriptions. - - Args: - model_name: The Ultravox model to use (ModelSize enum or string) - hf_token: Hugging Face token for model access - temperature: Sampling temperature for generation - max_tokens: Maximum tokens to generate - **kwargs: Additional arguments passed to AIService - - Attributes: - model: The UltravoxModel instance - buffer: Buffer to collect audio frames - temperature: Temperature for text generation - max_tokens: Maximum tokens to generate - _connection_active: Flag indicating if service is active + This service collects audio frames during speech and processes them with + Ultravox to generate text transcriptions. It handles real-time audio + buffering, model warm-up, and streaming text generation. """ def __init__( @@ -200,6 +184,17 @@ class UltravoxSTTService(AIService): max_tokens: int = 100, **kwargs, ): + """Initialize the UltravoxSTTService. + + Args: + model_name: The Ultravox model to use. Defaults to + "fixie-ai/ultravox-v0_5-llama-3_1-8b". + hf_token: Hugging Face token for model access. If None, will try + to use HF_TOKEN environment variable. + temperature: Sampling temperature for generation. Defaults to 0.7. + max_tokens: Maximum tokens to generate. Defaults to 100. + **kwargs: Additional arguments passed to AIService. + """ super().__init__(**kwargs) # Authenticate with Hugging Face if token provided @@ -283,8 +278,11 @@ class UltravoxSTTService(AIService): async def start(self, frame: StartFrame): """Handle service start. + Starts the service, marks it as active, and performs model warm-up + to ensure optimal performance for the first inference. + Args: - frame: StartFrame that triggered this method + frame: StartFrame that triggered this method. """ await super().start(frame) self._connection_active = True @@ -296,8 +294,10 @@ class UltravoxSTTService(AIService): async def stop(self, frame: EndFrame): """Handle service stop. + Stops the service and marks it as inactive. + Args: - frame: EndFrame that triggered this method + frame: EndFrame that triggered this method. """ await super().stop(frame) self._connection_active = False @@ -306,8 +306,10 @@ class UltravoxSTTService(AIService): async def cancel(self, frame: CancelFrame): """Handle service cancellation. + Cancels the service, clears any buffered audio, and marks it as inactive. + Args: - frame: CancelFrame that triggered this method + frame: CancelFrame that triggered this method. """ await super().cancel(frame) self._connection_active = False @@ -317,11 +319,12 @@ class UltravoxSTTService(AIService): async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames. - This method collects audio frames and processes them when speech ends. + This method collects audio frames during speech and processes them + when speech ends to generate text transcriptions. Args: - frame: The frame to process - direction: Direction of the frame (input/output) + frame: The frame to process. + direction: Direction of the frame (input/output). """ await super().process_frame(frame, direction) diff --git a/src/pipecat/services/vision_service.py b/src/pipecat/services/vision_service.py index d4314f874..e245e9a01 100644 --- a/src/pipecat/services/vision_service.py +++ b/src/pipecat/services/vision_service.py @@ -25,12 +25,14 @@ class VisionService(AIService): Provides common functionality for vision services that process images and generate textual responses. Handles image frame processing and integrates with the AI service infrastructure for metrics and lifecycle management. - - Args: - **kwargs: Additional arguments passed to the parent AIService. """ def __init__(self, **kwargs): + """Initialize the vision service. + + Args: + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._describe_text = None diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index 6fc8efb2e..a58a654d9 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -24,13 +24,15 @@ class WebsocketService(ABC): Provides websocket connection management, automatic reconnection with exponential backoff, connection verification, and error handling. Subclasses implement service-specific connection and message handling logic. - - Args: - reconnect_on_error: Whether to automatically reconnect on connection errors. - **kwargs: Additional arguments (unused, for compatibility). """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): + """Initialize the websocket service. + + Args: + reconnect_on_error: Whether to automatically reconnect on connection errors. + **kwargs: Additional arguments (unused, for compatibility). + """ self._websocket: Optional[websockets.WebSocketClientProtocol] = None self._reconnect_on_error = reconnect_on_error diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index 018dae85a..6f8ac26f2 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base class for Whisper-based speech-to-text services. + +This module provides common functionality for services implementing the Whisper API +interface, including language mapping, metrics generation, and error handling. +""" + from typing import AsyncGenerator, Optional from loguru import logger @@ -18,9 +24,16 @@ from pipecat.utils.tracing.service_decorators import traced_stt def language_to_whisper_language(language: Language) -> Optional[str]: - """Language support for Whisper API. + """Maps pipecat Language enum to Whisper API language codes. + Language support for Whisper API. Docs: https://platform.openai.com/docs/guides/speech-to-text#supported-languages + + Args: + language: A Language enum value representing the input language. + + Returns: + str or None: The corresponding Whisper language code, or None if not supported. """ BASE_LANGUAGES = { Language.AF: "af", @@ -98,15 +111,6 @@ class BaseWhisperSTTService(SegmentedSTTService): Provides common functionality for services implementing the Whisper API interface, including metrics generation and error handling. - - Args: - model: Name of the Whisper model to use. - api_key: Service API key. Defaults to None. - base_url: Service API base URL. Defaults to None. - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to SegmentedSTTService. """ def __init__( @@ -120,6 +124,17 @@ class BaseWhisperSTTService(SegmentedSTTService): temperature: Optional[float] = None, **kwargs, ): + """Initialize the Whisper STT service. + + Args: + model: Name of the Whisper model to use. + api_key: Service API key. Defaults to None. + base_url: Service API base URL. Defaults to None. + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ super().__init__(**kwargs) self.set_model_name(model) self._client = self._create_client(api_key, base_url) @@ -138,12 +153,30 @@ class BaseWhisperSTTService(SegmentedSTTService): return AsyncOpenAI(api_key=api_key, base_url=base_url) async def set_model(self, model: str): + """Set the model name for transcription. + + Args: + model: The name of the model to use. + """ self.set_model_name(model) def can_generate_metrics(self) -> bool: + """Indicates whether this service can generate metrics. + + Returns: + bool: True, as this service supports metric generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert from pipecat Language to service language code. + + Args: + language: The Language enum value to convert. + + Returns: + str or None: The corresponding service language code, or None if not supported. + """ return language_to_whisper_language(language) async def set_language(self, language: Language): @@ -163,6 +196,15 @@ class BaseWhisperSTTService(SegmentedSTTService): pass async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe audio data to text. + + Args: + audio: Raw audio data to transcribe. + + Yields: + Frame: Either a TranscriptionFrame containing the transcribed text + or an ErrorFrame if transcription fails. + """ try: await self.start_processing_metrics() await self.start_ttfb_metrics() diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index d6920ed6c..ace18ab56 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -4,7 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""This module implements Whisper transcription with a locally-downloaded model.""" +"""Whisper speech-to-text services with locally-downloaded models. + +This module implements Whisper transcription using locally-downloaded models, +supporting both Faster Whisper and MLX Whisper backends for efficient inference. +""" import asyncio from enum import Enum @@ -37,18 +41,18 @@ if TYPE_CHECKING: class Model(Enum): - """Class of basic Whisper model selection options. + """Whisper model selection options for Faster Whisper. - Available models: - Multilingual models: - TINY: Smallest multilingual model - BASE: Basic multilingual model - MEDIUM: Good balance for multilingual - LARGE: Best quality multilingual - DISTIL_LARGE_V2: Fast multilingual + Provides various model sizes and specializations for speech recognition, + balancing quality and performance based on use case requirements. - English-only models: - DISTIL_MEDIUM_EN: Fast English-only + Parameters: + TINY: Smallest multilingual model, fastest inference. + BASE: Basic multilingual model, good speed/quality balance. + MEDIUM: Medium-sized multilingual model, better quality. + LARGE: Best quality multilingual model, slower inference. + DISTIL_LARGE_V2: Fast multilingual distilled model. + DISTIL_MEDIUM_EN: Fast English-only distilled model. """ # Multilingual models @@ -63,16 +67,18 @@ class Model(Enum): class MLXModel(Enum): - """Class of MLX Whisper model selection options. + """MLX Whisper model selection options for Apple Silicon. - Available models: - Multilingual models: - TINY: Smallest multilingual model - MEDIUM: Good balance for multilingual - LARGE_V3: Best quality multilingual - LARGE_V3_TURBO: Finetuned, pruned Whisper large-v3, much faster, slightly lower quality - DISTIL_LARGE_V3: Fast multilingual - LARGE_V3_TURBO_Q4: LARGE_V3_TURBO, quantized to Q4 + Provides various model sizes optimized for Apple Silicon hardware, + including quantized variants for improved performance. + + Parameters: + TINY: Smallest multilingual model for MLX. + MEDIUM: Medium-sized multilingual model for MLX. + LARGE_V3: Best quality multilingual model for MLX. + LARGE_V3_TURBO: Finetuned, pruned Whisper large-v3, much faster with slightly lower quality. + DISTIL_LARGE_V3: Fast multilingual distilled model for MLX. + LARGE_V3_TURBO_Q4: LARGE_V3_TURBO quantized to Q4 for reduced memory usage. """ # Multilingual models @@ -256,21 +262,6 @@ class WhisperSTTService(SegmentedSTTService): This service uses Faster Whisper to perform speech-to-text transcription on audio segments. It supports multiple languages and various model sizes. - - Args: - model: The Whisper model to use for transcription. Can be a Model enum or string. - device: The device to run inference on ('cpu', 'cuda', or 'auto'). - compute_type: The compute type for inference ('default', 'int8', 'int8_float16', etc.). - no_speech_prob: Probability threshold for filtering out non-speech segments. - language: The default language for transcription. - **kwargs: Additional arguments passed to SegmentedSTTService. - - Attributes: - _device: The device used for inference. - _compute_type: The compute type for inference. - _no_speech_prob: Threshold for non-speech filtering. - _model: The loaded Whisper model instance. - _settings: Dictionary containing service settings. """ def __init__( @@ -283,6 +274,16 @@ class WhisperSTTService(SegmentedSTTService): language: Language = Language.EN, **kwargs, ): + """Initialize the Whisper STT service. + + Args: + model: The Whisper model to use for transcription. Can be a Model enum or string. + device: The device to run inference on ('cpu', 'cuda', or 'auto'). + compute_type: The compute type for inference ('default', 'int8', 'int8_float16', etc.). + no_speech_prob: Probability threshold for filtering out non-speech segments. + language: The default language for transcription. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ super().__init__(**kwargs) self._device: str = device self._compute_type = compute_type @@ -355,7 +356,7 @@ class WhisperSTTService(SegmentedSTTService): pass async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Transcribes given audio using Whisper. + """Transcribe audio data using Whisper. Args: audio: Raw audio bytes in 16-bit PCM format. @@ -402,18 +403,6 @@ class WhisperSTTServiceMLX(WhisperSTTService): This service uses MLX Whisper to perform speech-to-text transcription on audio segments. It's optimized for Apple Silicon and supports multiple languages and quantizations. - - Args: - model: The MLX Whisper model to use for transcription. Can be an MLXModel enum or string. - no_speech_prob: Probability threshold for filtering out non-speech segments. - language: The default language for transcription. - temperature: Temperature for sampling. Can be a float or tuple of floats. - **kwargs: Additional arguments passed to SegmentedSTTService. - - Attributes: - _no_speech_threshold: Threshold for non-speech filtering. - _temperature: Temperature for sampling. - _settings: Dictionary containing service settings. """ def __init__( @@ -425,6 +414,15 @@ class WhisperSTTServiceMLX(WhisperSTTService): temperature: float = 0.0, **kwargs, ): + """Initialize the MLX Whisper STT service. + + Args: + model: The MLX Whisper model to use for transcription. Can be an MLXModel enum or string. + no_speech_prob: Probability threshold for filtering out non-speech segments. + language: The default language for transcription. + temperature: Temperature for sampling. Can be a float or tuple of floats. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ # Skip WhisperSTTService.__init__ and call its parent directly SegmentedSTTService.__init__(self, **kwargs) @@ -455,7 +453,10 @@ class WhisperSTTServiceMLX(WhisperSTTService): @override async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Transcribes given audio using MLX Whisper. + """Transcribe audio data using MLX Whisper. + + The audio is expected to be 16-bit signed PCM data. + MLX Whisper will handle the conversion internally. Args: audio: Raw audio bytes in 16-bit PCM format. @@ -463,10 +464,6 @@ class WhisperSTTServiceMLX(WhisperSTTService): Yields: Frame: Either a TranscriptionFrame containing the transcribed text or an ErrorFrame if transcription fails. - - Note: - The audio is expected to be 16-bit signed PCM data. - MLX Whisper will handle the conversion internally. """ try: import mlx_whisper diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index 2111e4d72..6332e26ef 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""XTTS text-to-speech service implementation. + +This module provides integration with Coqui XTTS streaming server for +text-to-speech synthesis using local Docker deployment. +""" + from typing import Any, AsyncGenerator, Dict, Optional import aiohttp @@ -31,6 +37,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts def language_to_xtts_language(language: Language) -> Optional[str]: + """Convert a Language enum to XTTS language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding XTTS language code, or None if not supported. + """ BASE_LANGUAGES = { Language.CS: "cs", Language.DE: "de", @@ -70,6 +84,13 @@ def language_to_xtts_language(language: Language) -> Optional[str]: class XTTSService(TTSService): + """Coqui XTTS text-to-speech service. + + Provides text-to-speech synthesis using a locally running Coqui XTTS + streaming server. Supports multiple languages and voice cloning through + studio speakers configuration. + """ + def __init__( self, *, @@ -80,6 +101,16 @@ class XTTSService(TTSService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the XTTS service. + + Args: + voice_id: ID of the voice/speaker to use for synthesis. + base_url: Base URL of the XTTS streaming server. + aiohttp_session: HTTP session for making requests to the server. + language: Language for synthesis. Defaults to English. + sample_rate: Audio sample rate. If None, uses default. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._settings = { @@ -93,12 +124,30 @@ class XTTSService(TTSService): self._resampler = create_default_resampler() def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as XTTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to XTTS service language format. + + Args: + language: The language to convert. + + Returns: + The XTTS-specific language code, or None if not supported. + """ return language_to_xtts_language(language) async def start(self, frame: StartFrame): + """Start the XTTS service and load studio speakers. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) if self._studio_speakers: @@ -120,6 +169,14 @@ class XTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using XTTS streaming server. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") if not self._studio_speakers: diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index a6de4f46e..a2f269309 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -4,13 +4,23 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Language code enumerations for Pipecat. + +This module provides comprehensive language code constants following ISO 639 +and BCP 47 standards, supporting both language-only and language-region +combinations for various speech and text processing services. +""" + import sys from enum import Enum if sys.version_info < (3, 11): class StrEnum(str, Enum): + """String enumeration base class for Python < 3.11 compatibility.""" + def __new__(cls, value): + """Create a new instance of the StrEnum.""" obj = str.__new__(cls, value) obj._value_ = value return obj @@ -19,6 +29,14 @@ else: class Language(StrEnum): + """Language codes for speech and text processing services. + + Provides comprehensive language code constants following ISO 639 and BCP 47 + standards. Includes both language-only codes (e.g., 'en') and language-region + combinations (e.g., 'en-US') to support various speech synthesis, recognition, + and translation services. + """ + # Afrikaans AF = "af" AF_ZA = "af-ZA"