Merge pull request #2067 from pipecat-ai/mb/update-docstrings-for-ref-docs

Update base service class docstrings for better docs auto-generation
This commit is contained in:
Mark Backman
2025-06-26 07:07:59 -04:00
committed by GitHub
16 changed files with 1151 additions and 130 deletions

View File

@@ -41,36 +41,76 @@ We use Ruff for code linting and formatting. Please ensure your code passes all
We follow Google-style docstrings with these specific conventions: We follow Google-style docstrings with these specific conventions:
- Class docstrings should fully document all parameters used in `__init__` **Regular Classes:**
- We don't require separate docstrings for `__init__` methods when parameters are documented in the class docstring
- Property methods should have docstrings explaining their purpose and return value
Example of correctly documented class: - Class docstring describes the class purpose and documents all `__init__` parameters in an `Args:` section
- No separate `__init__` docstring needed
- All public methods must have docstrings with `Args:` and `Returns:` sections as appropriate
**Dataclasses:**
- Class docstring describes the purpose and documents all fields in a `Parameters:` section
- No `__init__` docstring (auto-generated)
**Properties:**
- Must have docstrings with `Returns:` section
**Abstract Methods:**
- Must have docstrings explaining what subclasses should implement
#### Examples:
```python ```python
class MyClass: # Regular class
"""Class description. class MyService(BaseService):
"""Description of what the service does.
Additional details about the class.
Args: Args:
param1: Description of first parameter. param1: Description of param1.
param2: Description of second parameter. param2: Description of param2. Defaults to True.
**kwargs: Additional arguments passed to parent.
""" """
def __init__(self, param1, param2): def __init__(self, param1: str, param2: bool = True, **kwargs):
# No docstring required here as parameters are documented above # No docstring - parameters documented above
self.param1 = param1 super().__init__(**kwargs)
self.param2 = param2
@property @property
def some_property(self) -> str: def sample_rate(self) -> int:
"""Get the formatted property value. """Get the current sample rate.
Returns: Returns:
A string representation of the property. The sample rate in Hz.
""" """
return f"Property: {self.param1}" return self._sample_rate
async def process_data(self, data: str) -> bool:
"""Process the provided data.
Args:
data: The data to process.
Returns:
True if processing succeeded.
"""
pass
# Dataclass
@dataclass
class ConfigParams:
"""Configuration parameters for the service.
Parameters:
host: The host address.
port: The port number. Defaults to 8080.
timeout: Connection timeout in seconds.
"""
host: str
port: int = 8080
timeout: float = 30.0
``` ```
# Contributor Covenant Code of Conduct # Contributor Covenant Code of Conduct

View File

@@ -1,5 +1,6 @@
import logging import logging
import sys import sys
from datetime import datetime
from pathlib import Path from pathlib import Path
# Configure logging # Configure logging
@@ -13,7 +14,8 @@ sys.path.insert(0, str(project_root / "src"))
# Project information # Project information
project = "pipecat-ai" project = "pipecat-ai"
copyright = "2024, Daily" current_year = datetime.now().year
copyright = f"2024-{current_year}, Daily" if current_year > 2024 else "2024, Daily"
author = "Daily" author = "Daily"
# General configuration # General configuration
@@ -27,15 +29,14 @@ extensions = [
# Napoleon settings # Napoleon settings
napoleon_google_docstring = True napoleon_google_docstring = True
napoleon_numpy_docstring = False napoleon_numpy_docstring = False
napoleon_include_init_with_doc = True napoleon_include_init_with_doc = False
# AutoDoc settings # AutoDoc settings
autodoc_default_options = { autodoc_default_options = {
"members": True, "members": True,
"member-order": "bysource", "member-order": "bysource",
"special-members": "__init__",
"undoc-members": True, "undoc-members": True,
"exclude-members": "__weakref__", "exclude-members": "__weakref__,__init__",
"no-index": True, "no-index": True,
"show-inheritance": True, "show-inheritance": True,
} }
@@ -145,6 +146,28 @@ autodoc_mock_imports = [
"transformers.AutoFeatureExtractor", "transformers.AutoFeatureExtractor",
# Also add specific classes that are imported # Also add specific classes that are imported
"AutoFeatureExtractor", "AutoFeatureExtractor",
# Sentry dependencies
"sentry_sdk",
# AWS Nova Sonic dependencies
"aws_sdk_bedrock_runtime",
"aws_sdk_bedrock_runtime.client",
"aws_sdk_bedrock_runtime.config",
"aws_sdk_bedrock_runtime.models",
"smithy_aws_core",
"smithy_aws_core.credentials_resolvers",
"smithy_aws_core.credentials_resolvers.static",
"smithy_aws_core.identity",
"smithy_core",
"smithy_core.aio",
"smithy_core.aio.eventstream",
# MCP dependencies (you may already have these)
"mcp",
"mcp.client",
"mcp.client.session_group",
"mcp.client.sse",
"mcp.client.stdio",
"mcp.ClientSession",
"mcp.StdioServerParameters",
] ]
# HTML output settings # HTML output settings
@@ -249,6 +272,9 @@ def clean_title(title: str) -> str:
"playht": "PlayHT", "playht": "PlayHT",
"xtts": "XTTS", "xtts": "XTTS",
"lmnt": "LMNT", "lmnt": "LMNT",
"stt": "STT",
"tts": "TTS",
"llm": "LLM",
} }
# Check if the entire title is a special case # Check if the entire title is a special case

View File

@@ -123,8 +123,7 @@ select = [
"D", # Docstring rules "D", # Docstring rules
"I", # Import rules "I", # Import rules
] ]
# We ignore D107 because class docstrings already document __init__ parameters # Ignore requirement for __init__ docstrings
# and our Sphinx configuration uses napoleon_include_init_with_doc=True
ignore = ["D107"] ignore = ["D107"]
[tool.ruff.lint.pydocstyle] [tool.ruff.lint.pydocstyle]

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Base AI service implementation.
Provides the foundation for all AI services in the Pipecat framework, including
model management, settings handling, and frame processing lifecycle methods.
"""
from typing import Any, AsyncGenerator, Dict, Mapping from typing import Any, AsyncGenerator, Dict, Mapping
from loguru import logger from loguru import logger
@@ -20,6 +26,17 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
class AIService(FrameProcessor): class AIService(FrameProcessor):
"""Base class for all AI services.
Provides common functionality for AI services including model management,
settings handling, session properties, and frame processing lifecycle.
Subclasses should implement specific AI functionality while leveraging
this base infrastructure.
Args:
**kwargs: Additional arguments passed to the parent FrameProcessor.
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
self._model_name: str = "" self._model_name: str = ""
@@ -28,19 +45,53 @@ class AIService(FrameProcessor):
@property @property
def model_name(self) -> str: def model_name(self) -> str:
"""Get the current model name.
Returns:
The name of the AI model being used.
"""
return self._model_name return self._model_name
def set_model_name(self, model: str): def set_model_name(self, model: str):
"""Set the AI model name and update metrics.
Args:
model: The name of the AI model to use.
"""
self._model_name = model self._model_name = model
self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name)) self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name))
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the AI service.
Called when the service should begin processing. Subclasses should
override this method to perform service-specific initialization.
Args:
frame: The start frame containing initialization parameters.
"""
pass pass
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the AI service.
Called when the service should stop processing. Subclasses should
override this method to perform cleanup operations.
Args:
frame: The end frame.
"""
pass pass
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the AI service.
Called when the service should cancel all operations. Subclasses should
override this method to handle cancellation logic.
Args:
frame: The cancel frame.
"""
pass pass
async def _update_settings(self, settings: Mapping[str, Any]): async def _update_settings(self, settings: Mapping[str, Any]):
@@ -87,6 +138,15 @@ class AIService(FrameProcessor):
logger.warning(f"Unknown setting for {self.name} service: {key}") logger.warning(f"Unknown setting for {self.name} service: {key}")
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames and handle service lifecycle.
Automatically handles StartFrame, EndFrame, and CancelFrame by calling
the appropriate lifecycle methods.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartFrame): if isinstance(frame, StartFrame):
@@ -97,6 +157,14 @@ class AIService(FrameProcessor):
await self.stop(frame) await self.stop(frame)
async def process_generator(self, generator: AsyncGenerator[Frame | None, None]): async def process_generator(self, generator: AsyncGenerator[Frame | None, None]):
"""Process frames from an async generator.
Takes an async generator that yields frames and processes each one,
handling error frames specially by pushing them as errors.
Args:
generator: An async generator that yields Frame objects or None.
"""
async for f in generator: async for f in generator:
if f: if f:
if isinstance(f, ErrorFrame): if isinstance(f, ErrorFrame):

View File

@@ -4,6 +4,17 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Deprecated AI services module.
This module is deprecated. Import services directly from their respective modules:
- pipecat.services.ai_service
- pipecat.services.image_service
- pipecat.services.llm_service
- pipecat.services.stt_service
- pipecat.services.tts_service
- pipecat.services.vision_service
"""
import sys import sys
from pipecat.services import DeprecatedModuleProxy from pipecat.services import DeprecatedModuleProxy

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Cartesia text-to-speech service implementations."""
import base64 import base64
import json import json
import uuid import uuid
@@ -43,6 +45,14 @@ except ModuleNotFoundError as e:
def language_to_cartesia_language(language: Language) -> Optional[str]: def language_to_cartesia_language(language: Language) -> Optional[str]:
"""Convert a Language enum to Cartesia language code.
Args:
language: The Language enum value to convert.
Returns:
The corresponding Cartesia language code, or None if not supported.
"""
BASE_LANGUAGES = { BASE_LANGUAGES = {
Language.DE: "de", Language.DE: "de",
Language.EN: "en", Language.EN: "en",
@@ -75,7 +85,35 @@ def language_to_cartesia_language(language: Language) -> Optional[str]:
class CartesiaTTSService(AudioContextWordTTSService): class CartesiaTTSService(AudioContextWordTTSService):
"""Cartesia TTS service with WebSocket streaming and word timestamps.
Provides text-to-speech using Cartesia's streaming WebSocket API.
Supports word-level timestamps, audio context management, and various voice
customization options including speed and emotion controls.
Args:
api_key: Cartesia API key for authentication.
voice_id: ID of the voice to use for synthesis.
cartesia_version: API version string for Cartesia service.
url: WebSocket URL for Cartesia TTS API.
model: TTS model to use (e.g., "sonic-2").
sample_rate: Audio sample rate. If None, uses default.
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
text_aggregator: Custom text aggregator for processing input text.
**kwargs: Additional arguments passed to the parent service.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Cartesia TTS configuration.
Parameters:
language: Language to use for synthesis.
speed: Voice speed control (string or float).
emotion: List of emotion controls (deprecated).
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed: Optional[Union[str, float]] = "" speed: Optional[Union[str, float]] = ""
emotion: Optional[List[str]] = [] emotion: Optional[List[str]] = []
@@ -138,14 +176,32 @@ class CartesiaTTSService(AudioContextWordTTSService):
self._receive_task = None self._receive_task = None
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Cartesia service supports metrics generation.
"""
return True return True
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the TTS model.
Args:
model: The model name to use for synthesis.
"""
self._model_id = model self._model_id = model
await super().set_model(model) await super().set_model(model)
logger.info(f"Switching TTS model to: [{model}]") logger.info(f"Switching TTS model to: [{model}]")
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Cartesia language format.
Args:
language: The language to convert.
Returns:
The Cartesia-specific language code, or None if not supported.
"""
return language_to_cartesia_language(language) return language_to_cartesia_language(language)
def _build_msg( def _build_msg(
@@ -183,15 +239,30 @@ class CartesiaTTSService(AudioContextWordTTSService):
return json.dumps(msg) return json.dumps(msg)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Cartesia TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate self._settings["output_format"]["sample_rate"] = self.sample_rate
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the Cartesia TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Stop the Cartesia TTS service.
Args:
frame: The end frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
@@ -248,6 +319,7 @@ class CartesiaTTSService(AudioContextWordTTSService):
self._context_id = None self._context_id = None
async def flush_audio(self): async def flush_audio(self):
"""Flush any pending audio and finalize the current context."""
if not self._context_id or not self._websocket: if not self._context_id or not self._websocket:
return return
logger.trace(f"{self}: flushing audio") logger.trace(f"{self}: flushing audio")
@@ -290,6 +362,14 @@ class CartesiaTTSService(AudioContextWordTTSService):
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Cartesia's streaming API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:
@@ -319,7 +399,34 @@ class CartesiaTTSService(AudioContextWordTTSService):
class CartesiaHttpTTSService(TTSService): class CartesiaHttpTTSService(TTSService):
"""Cartesia HTTP-based TTS service.
Provides text-to-speech using Cartesia's HTTP API for simpler, non-streaming
synthesis. Suitable for use cases where streaming is not required and simpler
integration is preferred.
Args:
api_key: Cartesia API key for authentication.
voice_id: ID of the voice to use for synthesis.
model: TTS model to use (e.g., "sonic-2").
base_url: Base URL for Cartesia HTTP API.
cartesia_version: API version string for Cartesia service.
sample_rate: Audio sample rate. If None, uses default.
encoding: Audio encoding format.
container: Audio container format.
params: Additional input parameters for voice customization.
**kwargs: Additional arguments passed to the parent TTSService.
"""
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for Cartesia HTTP TTS configuration.
Parameters:
language: Language to use for synthesis.
speed: Voice speed control (string or float).
emotion: List of emotion controls (deprecated).
"""
language: Optional[Language] = Language.EN language: Optional[Language] = Language.EN
speed: Optional[Union[str, float]] = "" speed: Optional[Union[str, float]] = ""
emotion: Optional[List[str]] = Field(default_factory=list) emotion: Optional[List[str]] = Field(default_factory=list)
@@ -366,25 +473,61 @@ class CartesiaHttpTTSService(TTSService):
) )
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Cartesia HTTP service supports metrics generation.
"""
return True return True
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a Language enum to Cartesia language format.
Args:
language: The language to convert.
Returns:
The Cartesia-specific language code, or None if not supported.
"""
return language_to_cartesia_language(language) return language_to_cartesia_language(language)
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Cartesia HTTP TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._settings["output_format"]["sample_rate"] = self.sample_rate self._settings["output_format"]["sample_rate"] = self.sample_rate
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the Cartesia HTTP TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._client.close() await self._client.close()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the Cartesia HTTP TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._client.close() await self._client.close()
@traced_tts @traced_tts
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Generate speech from text using Cartesia's HTTP API.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
logger.debug(f"{self}: Generating TTS [{text}]") logger.debug(f"{self}: Generating TTS [{text}]")
try: try:

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Deepgram speech-to-text service implementation."""
from typing import AsyncGenerator, Dict, Optional from typing import AsyncGenerator, Dict, Optional
from loguru import logger from loguru import logger
@@ -41,6 +43,22 @@ except ModuleNotFoundError as e:
class DeepgramSTTService(STTService): class DeepgramSTTService(STTService):
"""Deepgram speech-to-text service.
Provides real-time speech recognition using Deepgram's WebSocket API.
Supports configurable models, languages, VAD events, and various audio
processing options.
Args:
api_key: Deepgram API key for authentication.
url: Deprecated. Use base_url instead.
base_url: Custom Deepgram API base URL.
sample_rate: Audio sample rate. If None, uses default or live_options value.
live_options: Deepgram LiveOptions for detailed configuration.
addons: Additional Deepgram features to enable.
**kwargs: Additional arguments passed to the parent STTService.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -108,12 +126,27 @@ class DeepgramSTTService(STTService):
@property @property
def vad_enabled(self): def vad_enabled(self):
"""Check if Deepgram VAD events are enabled.
Returns:
True if VAD events are enabled in the current settings.
"""
return self._settings["vad_events"] return self._settings["vad_events"]
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as Deepgram service supports metrics generation.
"""
return True return True
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the Deepgram model and reconnect.
Args:
model: The Deepgram model name to use.
"""
await super().set_model(model) await super().set_model(model)
logger.info(f"Switching STT model to: [{model}]") logger.info(f"Switching STT model to: [{model}]")
self._settings["model"] = model self._settings["model"] = model
@@ -121,25 +154,53 @@ class DeepgramSTTService(STTService):
await self._connect() await self._connect()
async def set_language(self, language: Language): async def set_language(self, language: Language):
"""Set the recognition language and reconnect.
Args:
language: The language to use for speech recognition.
"""
logger.info(f"Switching STT language to: [{language}]") logger.info(f"Switching STT language to: [{language}]")
self._settings["language"] = language self._settings["language"] = language
await self._disconnect() await self._disconnect()
await self._connect() await self._connect()
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the Deepgram STT service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._settings["sample_rate"] = self.sample_rate self._settings["sample_rate"] = self.sample_rate
await self._connect() await self._connect()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the Deepgram STT service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._disconnect() await self._disconnect()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the Deepgram STT service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._disconnect() await self._disconnect()
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Send audio data to Deepgram for transcription.
Args:
audio: Raw audio bytes to transcribe.
Yields:
Frame: None (transcription results come via WebSocket callbacks).
"""
await self._connection.send(audio) await self._connection.send(audio)
yield None yield None
@@ -172,6 +233,7 @@ class DeepgramSTTService(STTService):
await self._connection.finish() await self._connection.finish()
async def start_metrics(self): async def start_metrics(self):
"""Start TTFB and processing metrics collection."""
await self.start_ttfb_metrics() await self.start_ttfb_metrics()
await self.start_processing_metrics() await self.start_processing_metrics()
@@ -235,6 +297,12 @@ class DeepgramSTTService(STTService):
) )
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with Deepgram-specific handling.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame) and not self.vad_enabled: if isinstance(frame, UserStartedSpeakingFrame) and not self.vad_enabled:

View File

@@ -4,6 +4,12 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Image generation service implementation.
Provides base functionality for AI-powered image generation services that convert
text prompts into images.
"""
from abc import abstractmethod from abc import abstractmethod
from typing import AsyncGenerator from typing import AsyncGenerator
@@ -13,15 +19,46 @@ from pipecat.services.ai_service import AIService
class ImageGenService(AIService): class ImageGenService(AIService):
"""Base class for image generation services.
Processes TextFrames by using their content as prompts for image generation.
Subclasses must implement the run_image_gen method to provide actual image
generation functionality using their specific AI service.
Args:
**kwargs: Additional arguments passed to the parent AIService.
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
# Renders the image. Returns an Image object. # Renders the image. Returns an Image object.
@abstractmethod @abstractmethod
async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]:
"""Generate an image from a text prompt.
This method must be implemented by subclasses to provide actual image
generation functionality using their specific AI service.
Args:
prompt: The text prompt to generate an image from.
Yields:
Frame: Frames containing the generated image (typically ImageRawFrame
or URLImageRawFrame).
"""
pass pass
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for image generation.
TextFrames are used as prompts for image generation, while other frames
are passed through unchanged.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, TextFrame): if isinstance(frame, TextFrame):

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Base classes for Large Language Model services with function calling support."""
import asyncio import asyncio
import inspect import inspect
from dataclasses import dataclass from dataclasses import dataclass
@@ -41,23 +43,34 @@ FunctionCallHandler = Callable[["FunctionCallParams"], Awaitable[None]]
# Type alias for a callback function that handles the result of an LLM function call. # Type alias for a callback function that handles the result of an LLM function call.
class FunctionCallResultCallback(Protocol): class FunctionCallResultCallback(Protocol):
"""Protocol for function call result callbacks.
Handles the result of an LLM function call execution.
"""
async def __call__( async def __call__(
self, result: Any, *, properties: Optional[FunctionCallResultProperties] = None self, result: Any, *, properties: Optional[FunctionCallResultProperties] = None
) -> None: ... ) -> None:
"""Call the result callback.
Args:
result: The result of the function call.
properties: Optional properties for the result.
"""
...
@dataclass @dataclass
class FunctionCallParams: class FunctionCallParams:
"""Parameters for a function call. """Parameters for a function call.
Attributes: Parameters:
function_name (str): The name of the function being called. function_name: The name of the function being called.
arguments (Mapping[str, Any]): The arguments for the function. tool_call_id: A unique identifier for the function call.
tool_call_id (str): A unique identifier for the function call. arguments: The arguments for the function.
llm (LLMService): The LLMService instance being used. llm: The LLMService instance being used.
context (OpenAILLMContext): The LLM context. context: The LLM context.
result_callback (FunctionCallResultCallback): Callback to handle the result of the function call. result_callback: Callback to handle the result of the function call.
""" """
function_name: str function_name: str
@@ -70,14 +83,14 @@ class FunctionCallParams:
@dataclass @dataclass
class FunctionCallRegistryItem: class FunctionCallRegistryItem:
"""Represents an entry in our function call registry. This is what the user """Represents an entry in the function call registry.
registers.
Attributes: This is what the user registers when calling register_function.
function_name (Optional[str]): The name of the function.
handler (FunctionCallHandler): The handler for processing function call parameters.
cancel_on_interruption (bool): Flag indicating whether to cancel the call on interruption.
Parameters:
function_name: The name of the function (None for catch-all handler).
handler: The handler for processing function call parameters.
cancel_on_interruption: Whether to cancel the call on interruption.
""" """
function_name: Optional[str] function_name: Optional[str]
@@ -87,16 +100,17 @@ class FunctionCallRegistryItem:
@dataclass @dataclass
class FunctionCallRunnerItem: class FunctionCallRunnerItem:
"""Represents an internal function call entry to our function call """Internal function call entry for the function call runner.
runner. The runner executes function calls in order.
Attributes: The runner executes function calls in order.
registry_name (Optional[str]): The function call name registration (could be None).
function_name (str): The name of the function.
tool_call_id (str): A unique identifier for the function call.
arguments (Mapping[str, Any]): The arguments for the function.
context (OpenAILLMContext): The LLM context.
Parameters:
registry_item: The registry item containing handler information.
function_name: The name of the function.
tool_call_id: A unique identifier for the function call.
arguments: The arguments for the function.
context: The LLM context.
run_llm: Optional flag to control LLM execution after function call.
""" """
registry_item: FunctionCallRegistryItem registry_item: FunctionCallRegistryItem
@@ -108,22 +122,32 @@ class FunctionCallRunnerItem:
class LLMService(AIService): class LLMService(AIService):
"""This is the base class for all LLM services. It handles function calling """Base class for all LLM services.
registration and execution. The class also provides event handlers.
An event to know when an LLM service completion timeout occurs: Handles function calling registration and execution with support for both
parallel and sequential execution modes. Provides event handlers for
completion timeouts and function call lifecycle events.
@task.event_handler("on_completion_timeout") Args:
async def on_completion_timeout(service): run_in_parallel: Whether to run function calls in parallel or sequentially.
... Defaults to True.
**kwargs: Additional arguments passed to the parent AIService.
And an event to know that function calls have been received from the LLM Event handlers:
service and that we are going to start executing them: on_completion_timeout: Called when an LLM completion timeout occurs.
on_function_calls_started: Called when function calls are received and
execution is about to start.
@task.event_handler("on_function_calls_started") Example:
async def on_function_calls_started(service, function_calls: Sequence[FunctionCallFromLLM]): ```python
... @task.event_handler("on_completion_timeout")
async def on_completion_timeout(service):
logger.warning("LLM completion timed out")
@task.event_handler("on_function_calls_started")
async def on_function_calls_started(service, function_calls):
logger.info(f"Starting {len(function_calls)} function calls")
```
""" """
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations. # OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
@@ -143,6 +167,11 @@ class LLMService(AIService):
self._register_event_handler("on_completion_timeout") self._register_event_handler("on_completion_timeout")
def get_llm_adapter(self) -> BaseLLMAdapter: def get_llm_adapter(self) -> BaseLLMAdapter:
"""Get the LLM adapter instance.
Returns:
The adapter instance used for LLM communication.
"""
return self._adapter return self._adapter
def create_context_aggregator( def create_context_aggregator(
@@ -152,24 +181,57 @@ class LLMService(AIService):
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> Any: ) -> Any:
"""Create a context aggregator for managing LLM conversation context.
Must be implemented by subclasses.
Args:
context: The LLM context to create an aggregator for.
user_params: Parameters for user message aggregation.
assistant_params: Parameters for assistant message aggregation.
Returns:
A context aggregator instance.
"""
pass pass
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the LLM service.
Args:
frame: The start frame.
"""
await super().start(frame) await super().start(frame)
if not self._run_in_parallel: if not self._run_in_parallel:
await self._create_sequential_runner_task() await self._create_sequential_runner_task()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the LLM service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
if not self._run_in_parallel: if not self._run_in_parallel:
await self._cancel_sequential_runner_task() await self._cancel_sequential_runner_task()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the LLM service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
if not self._run_in_parallel: if not self._run_in_parallel:
await self._cancel_sequential_runner_task() await self._cancel_sequential_runner_task()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process a frame.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, StartInterruptionFrame): if isinstance(frame, StartInterruptionFrame):
@@ -188,6 +250,18 @@ class LLMService(AIService):
*, *,
cancel_on_interruption: bool = True, cancel_on_interruption: bool = True,
): ):
"""Register a function handler for LLM function calls.
Args:
function_name: The name of the function to handle. Use None to handle
all function calls with a catch-all handler.
handler: The function handler. Should accept a single FunctionCallParams
parameter.
start_callback: Legacy callback function (deprecated). Put initialization
code at the top of your handler instead.
cancel_on_interruption: Whether to cancel this function call when an
interruption occurs. Defaults to True.
"""
# Registering a function with the function_name set to None will run # Registering a function with the function_name set to None will run
# that handler for all functions # that handler for all functions
self._functions[function_name] = FunctionCallRegistryItem( self._functions[function_name] = FunctionCallRegistryItem(
@@ -210,16 +284,38 @@ class LLMService(AIService):
self._start_callbacks[function_name] = start_callback self._start_callbacks[function_name] = start_callback
def unregister_function(self, function_name: Optional[str]): def unregister_function(self, function_name: Optional[str]):
"""Remove a registered function handler.
Args:
function_name: The name of the function handler to remove.
"""
del self._functions[function_name] del self._functions[function_name]
if self._start_callbacks[function_name]: if self._start_callbacks[function_name]:
del self._start_callbacks[function_name] del self._start_callbacks[function_name]
def has_function(self, function_name: str): def has_function(self, function_name: str):
"""Check if a function handler is registered.
Args:
function_name: The name of the function to check.
Returns:
True if the function is registered or if a catch-all handler (None)
is registered.
"""
if None in self._functions.keys(): if None in self._functions.keys():
return True return True
return function_name in self._functions.keys() return function_name in self._functions.keys()
async def run_function_calls(self, function_calls: Sequence[FunctionCallFromLLM]): async def run_function_calls(self, function_calls: Sequence[FunctionCallFromLLM]):
"""Execute a sequence of function calls from the LLM.
Triggers the on_function_calls_started event and executes functions
either in parallel or sequentially based on the run_in_parallel setting.
Args:
function_calls: The function calls to execute.
"""
if len(function_calls) == 0: if len(function_calls) == 0:
return return
@@ -257,7 +353,7 @@ class LLMService(AIService):
else: else:
await self._sequential_runner_queue.put(runner_item) await self._sequential_runner_queue.put(runner_item)
async def call_start_function(self, context: OpenAILLMContext, function_name: str): async def _call_start_function(self, context: OpenAILLMContext, function_name: str):
if function_name in self._start_callbacks.keys(): if function_name in self._start_callbacks.keys():
await self._start_callbacks[function_name](function_name, self, context) await self._start_callbacks[function_name](function_name, self, context)
elif None in self._start_callbacks.keys(): elif None in self._start_callbacks.keys():
@@ -272,6 +368,18 @@ class LLMService(AIService):
text_content: Optional[str] = None, text_content: Optional[str] = None,
video_source: Optional[str] = None, video_source: Optional[str] = None,
): ):
"""Request an image from a user.
Pushes a UserImageRequestFrame upstream to request an image from the
specified user.
Args:
user_id: The ID of the user to request an image from.
function_name: Optional function name associated with the request.
tool_call_id: Optional tool call ID associated with the request.
text_content: Optional text content/context for the image request.
video_source: Optional video source identifier.
"""
await self.push_frame( await self.push_frame(
UserImageRequestFrame( UserImageRequestFrame(
user_id=user_id, user_id=user_id,
@@ -316,7 +424,7 @@ class LLMService(AIService):
) )
# NOTE(aleix): This needs to be removed after we remove the deprecation. # NOTE(aleix): This needs to be removed after we remove the deprecation.
await self.call_start_function(runner_item.context, runner_item.function_name) await self._call_start_function(runner_item.context, runner_item.function_name)
# Push a function call in-progress downstream. This frame will let our # Push a function call in-progress downstream. This frame will let our
# assistant context aggregator know that we are in the middle of a # assistant context aggregator know that we are in the middle of a

View File

@@ -1,3 +1,11 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
"""MCP (Model Context Protocol) client for integrating external tools with LLMs."""
import json import json
from typing import Any, Dict, List, Optional, Union from typing import Any, Dict, List, Optional, Union
@@ -19,6 +27,20 @@ except ModuleNotFoundError as e:
class MCPClient(BaseObject): class MCPClient(BaseObject):
"""Client for Model Context Protocol (MCP) servers.
Enables integration with MCP servers to provide external tools and resources
to LLMs. Supports both stdio and SSE server connections with automatic tool
registration and schema conversion.
Args:
server_params: Server connection parameters (stdio or SSE).
**kwargs: Additional arguments passed to the parent BaseObject.
Raises:
TypeError: If server_params is not a supported parameter type.
"""
def __init__( def __init__(
self, self,
server_params: Union[StdioServerParameters, SseServerParameters], server_params: Union[StdioServerParameters, SseServerParameters],
@@ -39,6 +61,17 @@ class MCPClient(BaseObject):
) )
async def register_tools(self, llm) -> ToolsSchema: async def register_tools(self, llm) -> ToolsSchema:
"""Register all available MCP tools with an LLM service.
Connects to the MCP server, discovers available tools, converts their
schemas to Pipecat format, and registers them with the LLM service.
Args:
llm: The Pipecat LLM service to register tools with.
Returns:
A ToolsSchema containing all successfully registered tools.
"""
tools_schema = await self._register_tools(llm) tools_schema = await self._register_tools(llm)
return tools_schema return tools_schema
@@ -46,13 +79,13 @@ class MCPClient(BaseObject):
self, tool_name: str, tool_schema: Dict[str, Any] self, tool_name: str, tool_schema: Dict[str, Any]
) -> FunctionSchema: ) -> FunctionSchema:
"""Convert an mcp tool schema to Pipecat's FunctionSchema format. """Convert an mcp tool schema to Pipecat's FunctionSchema format.
Args: Args:
tool_name: The name of the tool tool_name: The name of the tool
tool_schema: The mcp tool schema tool_schema: The mcp tool schema
Returns: Returns:
A FunctionSchema instance A FunctionSchema instance
""" """
logger.debug(f"Converting schema for tool '{tool_name}'") logger.debug(f"Converting schema for tool '{tool_name}'")
logger.trace(f"Original schema: {json.dumps(tool_schema, indent=2)}") logger.trace(f"Original schema: {json.dumps(tool_schema, indent=2)}")
@@ -72,6 +105,7 @@ class MCPClient(BaseObject):
async def _sse_register_tools(self, llm) -> ToolsSchema: async def _sse_register_tools(self, llm) -> ToolsSchema:
"""Register all available mcp.run tools with the LLM service. """Register all available mcp.run tools with the LLM service.
Args: Args:
llm: The Pipecat LLM service to register tools with llm: The Pipecat LLM service to register tools with
Returns: Returns:
@@ -120,6 +154,7 @@ class MCPClient(BaseObject):
async def _stdio_register_tools(self, llm) -> ToolsSchema: async def _stdio_register_tools(self, llm) -> ToolsSchema:
"""Register all available mcp.run tools with the LLM service. """Register all available mcp.run tools with the LLM service.
Args: Args:
llm: The Pipecat LLM service to register tools with llm: The Pipecat LLM service to register tools with
Returns: Returns:

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Base OpenAI LLM service implementation."""
import base64 import base64
import json import json
from typing import Any, Dict, List, Mapping, Optional from typing import Any, Dict, List, Mapping, Optional
@@ -40,16 +42,39 @@ from pipecat.utils.watchdog_async_iterator import WatchdogAsyncIterator
class BaseOpenAILLMService(LLMService): class BaseOpenAILLMService(LLMService):
"""This is the base for all services that use the AsyncOpenAI client. """Base class for all services that use the AsyncOpenAI client.
This service consumes OpenAILLMContextFrame frames, which contain a reference This service consumes OpenAILLMContextFrame frames, which contain a reference
to an OpenAILLMContext frame. The OpenAILLMContext object defines the context to an OpenAILLMContext object. The context defines what is sent to the LLM for
sent to the LLM for a completion. This includes user, assistant and system messages completion, including user, assistant, and system messages, as well as tool
as well as tool choices and the tool, which is used if requesting function choices and function call configurations.
calls from the LLM.
Args:
model: The OpenAI model name to use (e.g., "gpt-4.1", "gpt-4o").
api_key: OpenAI API key. If None, uses environment variable.
base_url: Custom base URL for OpenAI API. If None, uses default.
organization: OpenAI organization ID.
project: OpenAI project ID.
default_headers: Additional HTTP headers to include in requests.
params: Input parameters for model configuration and behavior.
**kwargs: Additional arguments passed to the parent LLMService.
""" """
class InputParams(BaseModel): class InputParams(BaseModel):
"""Input parameters for OpenAI model configuration.
Parameters:
frequency_penalty: Penalty for frequent tokens (-2.0 to 2.0).
presence_penalty: Penalty for new tokens (-2.0 to 2.0).
seed: Random seed for deterministic outputs.
temperature: Sampling temperature (0.0 to 2.0).
top_k: Top-k sampling parameter (currently ignored by OpenAI).
top_p: Top-p (nucleus) sampling parameter (0.0 to 1.0).
max_tokens: Maximum tokens in response (deprecated, use max_completion_tokens).
max_completion_tokens: Maximum completion tokens to generate.
extra: Additional model-specific parameters.
"""
frequency_penalty: Optional[float] = Field( frequency_penalty: Optional[float] = Field(
default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0 default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0
) )
@@ -111,6 +136,19 @@ class BaseOpenAILLMService(LLMService):
default_headers=None, default_headers=None,
**kwargs, **kwargs,
): ):
"""Create an AsyncOpenAI client instance.
Args:
api_key: OpenAI API key.
base_url: Custom base URL for the API.
organization: OpenAI organization ID.
project: OpenAI project ID.
default_headers: Additional HTTP headers.
**kwargs: Additional client configuration arguments.
Returns:
Configured AsyncOpenAI client instance.
"""
return AsyncOpenAI( return AsyncOpenAI(
api_key=api_key, api_key=api_key,
base_url=base_url, base_url=base_url,
@@ -125,11 +163,25 @@ class BaseOpenAILLMService(LLMService):
) )
def can_generate_metrics(self) -> bool: def can_generate_metrics(self) -> bool:
"""Check if this service can generate processing metrics.
Returns:
True, as OpenAI service supports metrics generation.
"""
return True return True
async def get_chat_completions( async def get_chat_completions(
self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam]
) -> AsyncStream[ChatCompletionChunk]: ) -> AsyncStream[ChatCompletionChunk]:
"""Get streaming chat completions from OpenAI API.
Args:
context: The LLM context containing tools and configuration.
messages: List of chat completion messages to send.
Returns:
Async stream of chat completion chunks.
"""
params = { params = {
"model": self.model_name, "model": self.model_name,
"stream": True, "stream": True,
@@ -277,6 +329,15 @@ class BaseOpenAILLMService(LLMService):
await self.run_function_calls(function_calls) await self.run_function_calls(function_calls)
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for LLM completion requests.
Handles OpenAILLMContextFrame, LLMMessagesFrame, VisionImageRawFrame,
and LLMUpdateSettingsFrame to trigger LLM completions and manage settings.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
context = None context = None

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""OpenAI LLM service implementation with context aggregators."""
import json import json
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Optional from typing import Any, Optional
@@ -26,17 +28,46 @@ from pipecat.services.openai.base_llm import BaseOpenAILLMService
@dataclass @dataclass
class OpenAIContextAggregatorPair: class OpenAIContextAggregatorPair:
"""Pair of OpenAI context aggregators for user and assistant messages.
Parameters:
_user: User context aggregator for processing user messages.
_assistant: Assistant context aggregator for processing assistant messages.
"""
_user: "OpenAIUserContextAggregator" _user: "OpenAIUserContextAggregator"
_assistant: "OpenAIAssistantContextAggregator" _assistant: "OpenAIAssistantContextAggregator"
def user(self) -> "OpenAIUserContextAggregator": def user(self) -> "OpenAIUserContextAggregator":
"""Get the user context aggregator.
Returns:
The user context aggregator instance.
"""
return self._user return self._user
def assistant(self) -> "OpenAIAssistantContextAggregator": def assistant(self) -> "OpenAIAssistantContextAggregator":
"""Get the assistant context aggregator.
Returns:
The assistant context aggregator instance.
"""
return self._assistant return self._assistant
class OpenAILLMService(BaseOpenAILLMService): class OpenAILLMService(BaseOpenAILLMService):
"""OpenAI LLM service implementation.
Provides a complete OpenAI LLM service with context aggregation support.
Uses the BaseOpenAILLMService for core functionality and adds OpenAI-specific
context aggregator creation.
Args:
model: The OpenAI model name to use. Defaults to "gpt-4.1".
params: Input parameters for model configuration.
**kwargs: Additional arguments passed to the parent BaseOpenAILLMService.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -53,14 +84,15 @@ class OpenAILLMService(BaseOpenAILLMService):
user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(),
assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(),
) -> OpenAIContextAggregatorPair: ) -> OpenAIContextAggregatorPair:
"""Create an instance of OpenAIContextAggregatorPair from an """Create OpenAI-specific context aggregators.
OpenAILLMContext. Constructor keyword arguments for both the user and
assistant aggregators can be provided. Creates a pair of context aggregators optimized for OpenAI's message format,
including support for function calls, tool usage, and image handling.
Args: Args:
context (OpenAILLMContext): The LLM context. context: The LLM context to create aggregators for.
user_params (LLMUserAggregatorParams, optional): User aggregator parameters. user_params: Parameters for user message aggregation.
assistant_params (LLMAssistantAggregatorParams, optional): User aggregator parameters. assistant_params: Parameters for assistant message aggregation.
Returns: Returns:
OpenAIContextAggregatorPair: A pair of context aggregators, one for OpenAIContextAggregatorPair: A pair of context aggregators, one for
@@ -75,11 +107,32 @@ class OpenAILLMService(BaseOpenAILLMService):
class OpenAIUserContextAggregator(LLMUserContextAggregator): class OpenAIUserContextAggregator(LLMUserContextAggregator):
"""OpenAI-specific user context aggregator.
Handles aggregation of user messages for OpenAI LLM services.
Inherits all functionality from the base LLMUserContextAggregator.
"""
pass pass
class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
"""OpenAI-specific assistant context aggregator.
Handles aggregation of assistant messages for OpenAI LLM services,
with specialized support for OpenAI's function calling format,
tool usage tracking, and image message handling.
"""
async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame):
"""Handle a function call in progress.
Adds the function call to the context with an IN_PROGRESS status
to track ongoing function execution.
Args:
frame: Frame containing function call progress information.
"""
self._context.add_message( self._context.add_message(
{ {
"role": "assistant", "role": "assistant",
@@ -104,6 +157,14 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
) )
async def handle_function_call_result(self, frame: FunctionCallResultFrame): async def handle_function_call_result(self, frame: FunctionCallResultFrame):
"""Handle the result of a function call.
Updates the context with the function call result, replacing any
previous IN_PROGRESS status.
Args:
frame: Frame containing the function call result.
"""
if frame.result: if frame.result:
result = json.dumps(frame.result) result = json.dumps(frame.result)
await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) await self._update_function_call_result(frame.function_name, frame.tool_call_id, result)
@@ -113,6 +174,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
) )
async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame):
"""Handle a cancelled function call.
Updates the context to mark the function call as cancelled.
Args:
frame: Frame containing the function call cancellation information.
"""
await self._update_function_call_result( await self._update_function_call_result(
frame.function_name, frame.tool_call_id, "CANCELLED" frame.function_name, frame.tool_call_id, "CANCELLED"
) )
@@ -129,6 +197,14 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator):
message["content"] = result message["content"] = result
async def handle_user_image_frame(self, frame: UserImageRawFrame): async def handle_user_image_frame(self, frame: UserImageRawFrame):
"""Handle a user image frame from a function call request.
Marks the associated function call as completed and adds the image
to the context for processing.
Args:
frame: Frame containing the user image and request context.
"""
await self._update_function_call_result( await self._update_function_call_result(
frame.request.function_name, frame.request.tool_call_id, "COMPLETED" frame.request.function_name, frame.request.tool_call_id, "COMPLETED"
) )

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Base classes for Speech-to-Text services with continuous and segmented processing."""
import io import io
import wave import wave
from abc import abstractmethod from abc import abstractmethod
@@ -26,7 +28,19 @@ from pipecat.transcriptions.language import Language
class STTService(AIService): class STTService(AIService):
"""STTService is a base class for speech-to-text services.""" """Base class for speech-to-text services.
Provides common functionality for STT services including audio passthrough,
muting, settings management, and audio processing. Subclasses must implement
the run_stt method to provide actual speech recognition.
Args:
audio_passthrough: Whether to pass audio frames downstream after processing.
Defaults to True.
sample_rate: The sample rate for audio input. If None, will be determined
from the start frame.
**kwargs: Additional arguments passed to the parent AIService.
"""
def __init__( def __init__(
self, self,
@@ -44,25 +58,59 @@ class STTService(AIService):
@property @property
def is_muted(self) -> bool: def is_muted(self) -> bool:
"""Returns whether the STT service is currently muted.""" """Check if the STT service is currently muted.
Returns:
True if the service is muted and will not process audio.
"""
return self._muted return self._muted
@property @property
def sample_rate(self) -> int: def sample_rate(self) -> int:
"""Get the current sample rate for audio processing.
Returns:
The sample rate in Hz.
"""
return self._sample_rate return self._sample_rate
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the speech recognition model.
Args:
model: The name of the model to use for speech recognition.
"""
self.set_model_name(model) self.set_model_name(model)
async def set_language(self, language: Language): async def set_language(self, language: Language):
"""Set the language for speech recognition.
Args:
language: The language to use for speech recognition.
"""
pass pass
@abstractmethod @abstractmethod
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
"""Returns transcript as a string""" """Run speech-to-text on the provided audio data.
This method must be implemented by subclasses to provide actual speech
recognition functionality.
Args:
audio: Raw audio bytes to transcribe.
Yields:
Frame: Frames containing transcription results (typically TextFrame).
"""
pass pass
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the STT service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._sample_rate = self._init_sample_rate or frame.audio_in_sample_rate self._sample_rate = self._init_sample_rate or frame.audio_in_sample_rate
@@ -80,13 +128,24 @@ class STTService(AIService):
logger.warning(f"Unknown setting for STT service: {key}") logger.warning(f"Unknown setting for STT service: {key}")
async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
"""Process an audio frame for speech recognition.
Args:
frame: The audio frame to process.
direction: The direction of frame processing.
"""
if self._muted: if self._muted:
return return
await self.process_generator(self.run_stt(frame.audio)) await self.process_generator(self.run_stt(frame.audio))
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Processes a frame of audio data, either buffering or transcribing it.""" """Process frames, handling VAD events and audio segmentation.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, AudioRawFrame): if isinstance(frame, AudioRawFrame):
@@ -106,14 +165,19 @@ class STTService(AIService):
class SegmentedSTTService(STTService): class SegmentedSTTService(STTService):
"""SegmentedSTTService is an STTService that uses VAD events to detect """STT service that processes speech in segments using VAD events.
speech and will run speech-to-text on speech segments only, instead of a
continous stream. Since it uses VAD it means that VAD needs to be enabled in
the pipeline.
This service always keeps a small audio buffer to take into account that VAD Uses Voice Activity Detection (VAD) events to detect speech segments and runs
events are delayed from when the user speech really starts. speech-to-text only on those segments, rather than continuously.
Requires VAD to be enabled in the pipeline to function properly. Maintains a
small audio buffer to account for the delay between actual speech start and
VAD detection.
Args:
sample_rate: The sample rate for audio input. If None, will be determined
from the start frame.
**kwargs: Additional arguments passed to the parent STTService.
""" """
def __init__(self, *, sample_rate: Optional[int] = None, **kwargs): def __init__(self, *, sample_rate: Optional[int] = None, **kwargs):
@@ -125,10 +189,16 @@ class SegmentedSTTService(STTService):
self._user_speaking = False self._user_speaking = False
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the segmented STT service and initialize audio buffer.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._audio_buffer_size_1s = self.sample_rate * 2 self._audio_buffer_size_1s = self.sample_rate * 2
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames, handling VAD events and audio segmentation."""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, UserStartedSpeakingFrame):
@@ -162,6 +232,15 @@ class SegmentedSTTService(STTService):
self._audio_buffer.clear() self._audio_buffer.clear()
async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection):
"""Process audio frames by buffering them for segmented transcription.
Continuously buffers audio, growing the buffer while user is speaking and
maintaining a small buffer when not speaking to account for VAD delay.
Args:
frame: The audio frame to process.
direction: The direction of frame processing.
"""
# If the user is speaking the audio buffer will keep growing. # If the user is speaking the audio buffer will keep growing.
self._audio_buffer += frame.audio self._audio_buffer += frame.audio

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Base classes for Text-to-speech services."""
import asyncio import asyncio
from abc import abstractmethod from abc import abstractmethod
from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple
@@ -43,6 +45,28 @@ from pipecat.utils.watchdog_queue import WatchdogQueue
class TTSService(AIService): class TTSService(AIService):
"""Base class for text-to-speech services.
Provides common functionality for TTS services including text aggregation,
filtering, audio generation, and frame management. Supports configurable
sentence aggregation, silence insertion, and frame processing control.
Args:
aggregate_sentences: Whether to aggregate text into sentences before synthesis.
push_text_frames: Whether to push TextFrames and LLMFullResponseEndFrames.
push_stop_frames: Whether to automatically push TTSStoppedFrames.
stop_frame_timeout_s: Idle time before pushing TTSStoppedFrame when push_stop_frames is True.
push_silence_after_stop: Whether to push silence audio after TTSStoppedFrame.
silence_time_s: Duration of silence to push when push_silence_after_stop is True.
pause_frame_processing: Whether to pause frame processing during audio generation.
sample_rate: Output sample rate for generated audio.
text_aggregator: Custom text aggregator for processing incoming text.
text_filters: Sequence of text filters to apply after aggregation.
text_filter: Single text filter (deprecated, use text_filters).
transport_destination: Destination for generated audio frames.
**kwargs: Additional arguments passed to the parent AIService.
"""
def __init__( def __init__(
self, self,
*, *,
@@ -105,54 +129,113 @@ class TTSService(AIService):
@property @property
def sample_rate(self) -> int: def sample_rate(self) -> int:
"""Get the current sample rate for audio output.
Returns:
The sample rate in Hz.
"""
return self._sample_rate return self._sample_rate
@property @property
def chunk_size(self) -> int: def chunk_size(self) -> int:
"""This property indicates how much audio we download (from TTS services """Get the recommended chunk size for audio streaming.
This property indicates how much audio we download (from TTS services
that require chunking) before we start pushing the first audio that require chunking) before we start pushing the first audio
frame. This will make sure we download the rest of the audio while audio frame. This will make sure we download the rest of the audio while audio
is being played without causing audio glitches (specially at the is being played without causing audio glitches (specially at the
beginning). Of course, this will also depend on how fast the TTS service beginning). Of course, this will also depend on how fast the TTS service
generates bytes. generates bytes.
Returns:
The recommended chunk size in bytes.
""" """
CHUNK_SECONDS = 0.5 CHUNK_SECONDS = 0.5
return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample
async def set_model(self, model: str): async def set_model(self, model: str):
"""Set the TTS model to use.
Args:
model: The name of the TTS model.
"""
self.set_model_name(model) self.set_model_name(model)
def set_voice(self, voice: str): def set_voice(self, voice: str):
"""Set the voice for speech synthesis.
Args:
voice: The voice identifier or name.
"""
self._voice_id = voice self._voice_id = voice
# Converts the text to audio. # Converts the text to audio.
@abstractmethod @abstractmethod
async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]:
"""Run text-to-speech synthesis on the provided text.
This method must be implemented by subclasses to provide actual TTS functionality.
Args:
text: The text to synthesize into speech.
Yields:
Frame: Audio frames containing the synthesized speech.
"""
pass pass
def language_to_service_language(self, language: Language) -> Optional[str]: def language_to_service_language(self, language: Language) -> Optional[str]:
"""Convert a language to the service-specific language format.
Args:
language: The language to convert.
Returns:
The service-specific language identifier, or None if not supported.
"""
return Language(language) return Language(language)
async def update_setting(self, key: str, value: Any): async def update_setting(self, key: str, value: Any):
"""Update a service-specific setting.
Args:
key: The setting key to update.
value: The new value for the setting.
"""
pass pass
async def flush_audio(self): async def flush_audio(self):
"""Flush any buffered audio data."""
pass pass
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate
if self._push_stop_frames and not self._stop_frame_task: if self._push_stop_frames and not self._stop_frame_task:
self._stop_frame_task = self.create_task(self._stop_frame_handler()) self._stop_frame_task = self.create_task(self._stop_frame_handler())
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
if self._stop_frame_task: if self._stop_frame_task:
await self.cancel_task(self._stop_frame_task) await self.cancel_task(self._stop_frame_task)
self._stop_frame_task = None self._stop_frame_task = None
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
if self._stop_frame_task: if self._stop_frame_task:
await self.cancel_task(self._stop_frame_task) await self.cancel_task(self._stop_frame_task)
@@ -176,9 +259,23 @@ class TTSService(AIService):
logger.warning(f"Unknown setting for TTS service: {key}") logger.warning(f"Unknown setting for TTS service: {key}")
async def say(self, text: str): async def say(self, text: str):
"""Immediately speak the provided text.
Args:
text: The text to speak.
"""
await self.queue_frame(TTSSpeakFrame(text)) await self.queue_frame(TTSSpeakFrame(text))
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames for text-to-speech conversion.
Handles TextFrames for synthesis, interruption frames, settings updates,
and various control frames.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if ( if (
@@ -223,6 +320,12 @@ class TTSService(AIService):
await self.push_frame(frame, direction) await self.push_frame(frame, direction)
async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM):
"""Push a frame downstream with TTS-specific handling.
Args:
frame: The frame to push.
direction: The direction to push the frame.
"""
if self._push_silence_after_stop and isinstance(frame, TTSStoppedFrame): if self._push_silence_after_stop and isinstance(frame, TTSStoppedFrame):
silence_num_bytes = int(self._silence_time_s * self.sample_rate * 2) # 16-bit silence_num_bytes = int(self._silence_time_s * self.sample_rate * 2) # 16-bit
silence_frame = TTSAudioRawFrame( silence_frame = TTSAudioRawFrame(
@@ -321,10 +424,13 @@ class TTSService(AIService):
class WordTTSService(TTSService): class WordTTSService(TTSService):
"""This is a base class for TTS services that support word timestamps. Word """Base class for TTS services that support word timestamps.
timestamps are useful to synchronize audio with text of the spoken
Word timestamps are useful to synchronize audio with text of the spoken
words. This way only the spoken words are added to the conversation context. words. This way only the spoken words are added to the conversation context.
Args:
**kwargs: Additional arguments passed to the parent TTSService.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
@@ -334,29 +440,57 @@ class WordTTSService(TTSService):
self._llm_response_started: bool = False self._llm_response_started: bool = False
def start_word_timestamps(self): def start_word_timestamps(self):
"""Start tracking word timestamps from the current time."""
if self._initial_word_timestamp == -1: if self._initial_word_timestamp == -1:
self._initial_word_timestamp = self.get_clock().get_time() self._initial_word_timestamp = self.get_clock().get_time()
def reset_word_timestamps(self): def reset_word_timestamps(self):
"""Reset word timestamp tracking."""
self._initial_word_timestamp = -1 self._initial_word_timestamp = -1
async def add_word_timestamps(self, word_times: List[Tuple[str, float]]): async def add_word_timestamps(self, word_times: List[Tuple[str, float]]):
"""Add word timestamps to the processing queue.
Args:
word_times: List of (word, timestamp) tuples where timestamp is in seconds.
"""
for word, timestamp in word_times: for word, timestamp in word_times:
await self._words_queue.put((word, seconds_to_nanoseconds(timestamp))) await self._words_queue.put((word, seconds_to_nanoseconds(timestamp)))
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the word TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._create_words_task() self._create_words_task()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the word TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
await self._stop_words_task() await self._stop_words_task()
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the word TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._stop_words_task() await self._stop_words_task()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with word timestamp awareness.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, LLMFullResponseStartFrame): if isinstance(frame, LLMFullResponseStartFrame):
@@ -403,15 +537,24 @@ class WordTTSService(TTSService):
class WebsocketTTSService(TTSService, WebsocketService): class WebsocketTTSService(TTSService, WebsocketService):
"""This is a base class for websocket-based TTS services. """Base class for websocket-based TTS services.
If an error occurs with the websocket, an "on_connection_error" event will Combines TTS functionality with websocket connectivity, providing automatic
be triggered: error handling and reconnection capabilities.
@tts.event_handler("on_connection_error") Args:
async def on_connection_error(tts: TTSService, error: str): reconnect_on_error: Whether to automatically reconnect on websocket errors.
... **kwargs: Additional arguments passed to parent classes.
Event handlers:
on_connection_error: Called when a websocket connection error occurs.
Example:
```python
@tts.event_handler("on_connection_error")
async def on_connection_error(tts: TTSService, error: str):
logger.error(f"TTS connection error: {error}")
```
""" """
def __init__(self, *, reconnect_on_error: bool = True, **kwargs): def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
@@ -425,10 +568,13 @@ class WebsocketTTSService(TTSService, WebsocketService):
class InterruptibleTTSService(WebsocketTTSService): class InterruptibleTTSService(WebsocketTTSService):
"""This is a base class for websocket-based TTS services that don't support """Websocket-based TTS service that handles interruptions without word timestamps.
word timestamps and that don't offer a way to correlate the generated audio
to the requested text.
Designed for TTS services that don't support word timestamps. Handles interruptions
by reconnecting the websocket when the bot is speaking and gets interrupted.
Args:
**kwargs: Additional arguments passed to the parent WebsocketTTSService.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
@@ -446,6 +592,12 @@ class InterruptibleTTSService(WebsocketTTSService):
await self._connect() await self._connect()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with bot speaking state tracking.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, BotStartedSpeakingFrame): if isinstance(frame, BotStartedSpeakingFrame):
@@ -455,16 +607,23 @@ class InterruptibleTTSService(WebsocketTTSService):
class WebsocketWordTTSService(WordTTSService, WebsocketService): class WebsocketWordTTSService(WordTTSService, WebsocketService):
"""This is a base class for websocket-based TTS services that support word """Base class for websocket-based TTS services that support word timestamps.
timestamps.
If an error occurs with the websocket a "on_connection_error" event will be Combines word timestamp functionality with websocket connectivity.
triggered:
@tts.event_handler("on_connection_error") Args:
async def on_connection_error(tts: TTSService, error: str): reconnect_on_error: Whether to automatically reconnect on websocket errors.
... **kwargs: Additional arguments passed to parent classes.
Event handlers:
on_connection_error: Called when a websocket connection error occurs.
Example:
```python
@tts.event_handler("on_connection_error")
async def on_connection_error(tts: TTSService, error: str):
logger.error(f"TTS connection error: {error}")
```
""" """
def __init__(self, *, reconnect_on_error: bool = True, **kwargs): def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
@@ -478,10 +637,13 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService):
class InterruptibleWordTTSService(WebsocketWordTTSService): class InterruptibleWordTTSService(WebsocketWordTTSService):
"""This is a base class for websocket-based TTS services that support word """Websocket-based TTS service with word timestamps that handles interruptions.
timestamps but don't offer a way to correlate the generated audio to the
requested text.
For TTS services that support word timestamps but can't correlate generated
audio with requested text. Handles interruptions by reconnecting when needed.
Args:
**kwargs: Additional arguments passed to the parent WebsocketWordTTSService.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
@@ -499,6 +661,12 @@ class InterruptibleWordTTSService(WebsocketWordTTSService):
await self._connect() await self._connect()
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames with bot speaking state tracking.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, BotStartedSpeakingFrame): if isinstance(frame, BotStartedSpeakingFrame):
@@ -508,7 +676,9 @@ class InterruptibleWordTTSService(WebsocketWordTTSService):
class AudioContextWordTTSService(WebsocketWordTTSService): class AudioContextWordTTSService(WebsocketWordTTSService):
"""This is a base class for websocket-based TTS services that support word """Websocket-based TTS service with word timestamps and audio context management.
This is a base class for websocket-based TTS services that support word
timestamps and also allow correlating the generated audio with the requested timestamps and also allow correlating the generated audio with the requested
text. text.
@@ -520,6 +690,8 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
we requested audio for a context "A" and then audio for context "B", the we requested audio for a context "A" and then audio for context "B", the
audio from context ID "A" will be played first. audio from context ID "A" will be played first.
Args:
**kwargs: Additional arguments passed to the parent WebsocketWordTTSService.
""" """
def __init__(self, **kwargs): def __init__(self, **kwargs):
@@ -528,13 +700,22 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
self._audio_context_task = None self._audio_context_task = None
async def create_audio_context(self, context_id: str): async def create_audio_context(self, context_id: str):
"""Create a new audio context.""" """Create a new audio context for grouping related audio.
Args:
context_id: Unique identifier for the audio context.
"""
await self._contexts_queue.put(context_id) await self._contexts_queue.put(context_id)
self._contexts[context_id] = asyncio.Queue() self._contexts[context_id] = asyncio.Queue()
logger.trace(f"{self} created audio context {context_id}") logger.trace(f"{self} created audio context {context_id}")
async def append_to_audio_context(self, context_id: str, frame: TTSAudioRawFrame): async def append_to_audio_context(self, context_id: str, frame: TTSAudioRawFrame):
"""Append audio to an existing context.""" """Append audio to an existing context.
Args:
context_id: The context to append audio to.
frame: The audio frame to append.
"""
if self.audio_context_available(context_id): if self.audio_context_available(context_id):
logger.trace(f"{self} appending audio {frame} to audio context {context_id}") logger.trace(f"{self} appending audio {frame} to audio context {context_id}")
await self._contexts[context_id].put(frame) await self._contexts[context_id].put(frame)
@@ -542,7 +723,11 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
logger.warning(f"{self} unable to append audio to context {context_id}") logger.warning(f"{self} unable to append audio to context {context_id}")
async def remove_audio_context(self, context_id: str): async def remove_audio_context(self, context_id: str):
"""Remove an existing audio context.""" """Remove an existing audio context.
Args:
context_id: The context to remove.
"""
if self.audio_context_available(context_id): if self.audio_context_available(context_id):
# We just mark the audio context for deletion by appending # We just mark the audio context for deletion by appending
# None. Once we reach None while handling audio we know we can # None. Once we reach None while handling audio we know we can
@@ -553,14 +738,31 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
logger.warning(f"{self} unable to remove context {context_id}") logger.warning(f"{self} unable to remove context {context_id}")
def audio_context_available(self, context_id: str) -> bool: def audio_context_available(self, context_id: str) -> bool:
"""Checks whether the given audio context is registered.""" """Check whether the given audio context is registered.
Args:
context_id: The context ID to check.
Returns:
True if the context exists and is available.
"""
return context_id in self._contexts return context_id in self._contexts
async def start(self, frame: StartFrame): async def start(self, frame: StartFrame):
"""Start the audio context TTS service.
Args:
frame: The start frame containing initialization parameters.
"""
await super().start(frame) await super().start(frame)
self._create_audio_context_task() self._create_audio_context_task()
async def stop(self, frame: EndFrame): async def stop(self, frame: EndFrame):
"""Stop the audio context TTS service.
Args:
frame: The end frame.
"""
await super().stop(frame) await super().stop(frame)
if self._audio_context_task: if self._audio_context_task:
# Indicate no more audio contexts are available. this will end the # Indicate no more audio contexts are available. this will end the
@@ -570,6 +772,11 @@ class AudioContextWordTTSService(WebsocketWordTTSService):
self._audio_context_task = None self._audio_context_task = None
async def cancel(self, frame: CancelFrame): async def cancel(self, frame: CancelFrame):
"""Cancel the audio context TTS service.
Args:
frame: The cancel frame.
"""
await super().cancel(frame) await super().cancel(frame)
await self._stop_audio_context_task() await self._stop_audio_context_task()

View File

@@ -4,6 +4,13 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Vision service implementation.
Provides base classes and implementations for computer vision services that can
analyze images and generate textual descriptions or answers to questions about
visual content.
"""
from abc import abstractmethod from abc import abstractmethod
from typing import AsyncGenerator from typing import AsyncGenerator
@@ -13,7 +20,15 @@ from pipecat.services.ai_service import AIService
class VisionService(AIService): class VisionService(AIService):
"""VisionService is a base class for vision services.""" """Base class for vision services.
Provides common functionality for vision services that process images and
generate textual responses. Handles image frame processing and integrates
with the AI service infrastructure for metrics and lifecycle management.
Args:
**kwargs: Additional arguments passed to the parent AIService.
"""
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
@@ -21,9 +36,31 @@ class VisionService(AIService):
@abstractmethod @abstractmethod
async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]:
"""Process a vision image frame and generate results.
This method must be implemented by subclasses to provide actual computer
vision functionality such as image description, object detection, or
visual question answering.
Args:
frame: The vision image frame to process, containing image data.
Yields:
Frame: Frames containing the vision analysis results, typically TextFrame
objects with descriptions or answers.
"""
pass pass
async def process_frame(self, frame: Frame, direction: FrameDirection): async def process_frame(self, frame: Frame, direction: FrameDirection):
"""Process frames, handling vision image frames for analysis.
Automatically processes VisionImageRawFrame objects by calling run_vision
and handles metrics tracking. Other frames are passed through unchanged.
Args:
frame: The frame to process.
direction: The direction of frame processing.
"""
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, VisionImageRawFrame): if isinstance(frame, VisionImageRawFrame):

View File

@@ -4,6 +4,8 @@
# SPDX-License-Identifier: BSD 2-Clause License # SPDX-License-Identifier: BSD 2-Clause License
# #
"""Base websocket service with automatic reconnection and error handling."""
import asyncio import asyncio
from abc import ABC, abstractmethod from abc import ABC, abstractmethod
from typing import Awaitable, Callable, Optional from typing import Awaitable, Callable, Optional
@@ -17,18 +19,26 @@ from pipecat.utils.network import exponential_backoff_time
class WebsocketService(ABC): class WebsocketService(ABC):
"""Base class for websocket-based services with reconnection logic.""" """Base class for websocket-based services with automatic reconnection.
Provides websocket connection management, automatic reconnection with
exponential backoff, connection verification, and error handling.
Subclasses implement service-specific connection and message handling logic.
Args:
reconnect_on_error: Whether to automatically reconnect on connection errors.
**kwargs: Additional arguments (unused, for compatibility).
"""
def __init__(self, *, reconnect_on_error: bool = True, **kwargs): def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
"""Initialize websocket attributes."""
self._websocket: Optional[websockets.WebSocketClientProtocol] = None self._websocket: Optional[websockets.WebSocketClientProtocol] = None
self._reconnect_on_error = reconnect_on_error self._reconnect_on_error = reconnect_on_error
async def _verify_connection(self) -> bool: async def _verify_connection(self) -> bool:
"""Verify websocket connection is working. """Verify the websocket connection is active and responsive.
Returns: Returns:
bool: True if connection is verified working, False otherwise True if connection is verified working, False otherwise.
""" """
try: try:
if not self._websocket or self._websocket.closed: if not self._websocket or self._websocket.closed:
@@ -40,13 +50,13 @@ class WebsocketService(ABC):
return False return False
async def _reconnect_websocket(self, attempt_number: int) -> bool: async def _reconnect_websocket(self, attempt_number: int) -> bool:
"""Reconnect the websocket. """Reconnect the websocket with the current attempt number.
Args: Args:
attempt_number: Current retry attempt number attempt_number: Current retry attempt number for logging.
Returns: Returns:
bool: True if reconnection and verification successful, False otherwise True if reconnection and verification successful, False otherwise.
""" """
logger.warning(f"{self} reconnecting (attempt: {attempt_number})") logger.warning(f"{self} reconnecting (attempt: {attempt_number})")
await self._disconnect_websocket() await self._disconnect_websocket()
@@ -54,10 +64,14 @@ class WebsocketService(ABC):
return await self._verify_connection() return await self._verify_connection()
async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]): async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]):
"""Handles WebSocket message receiving with automatic retry logic. """Handle websocket message receiving with automatic retry logic.
Continuously receives messages with automatic reconnection on errors.
Uses exponential backoff between retry attempts and reports fatal errors
after maximum retries are exhausted.
Args: Args:
report_error: Callback to report errors report_error: Callback function to report connection errors.
""" """
retry_count = 0 retry_count = 0
MAX_RETRIES = 3 MAX_RETRIES = 3
@@ -98,33 +112,45 @@ class WebsocketService(ABC):
@abstractmethod @abstractmethod
async def _connect(self): async def _connect(self):
"""Implement service-specific connection logic. This function will """Connect to the service.
connect to the websocket via _connect_websocket() among other connection
logic.""" Implement service-specific connection logic including websocket connection
via _connect_websocket() and any additional setup required.
"""
pass pass
@abstractmethod @abstractmethod
async def _disconnect(self): async def _disconnect(self):
"""Implement service-specific disconnection logic. This function will """Disconnect from the service.
disconnect to the websocket via _connect_websocket() among other
connection logic.
Implement service-specific disconnection logic including websocket
disconnection via _disconnect_websocket() and any cleanup required.
""" """
pass pass
@abstractmethod @abstractmethod
async def _connect_websocket(self): async def _connect_websocket(self):
"""Implement service-specific websocket connection logic. This function """Establish the websocket connection.
should only connect to the websocket."""
Implement the low-level websocket connection logic specific to the service.
Should only handle websocket connection, not additional service setup.
"""
pass pass
@abstractmethod @abstractmethod
async def _disconnect_websocket(self): async def _disconnect_websocket(self):
"""Implement service-specific websocket disconnection logic. This """Close the websocket connection.
function should only disconnect from the websocket."""
Implement the low-level websocket disconnection logic specific to the service.
Should only handle websocket disconnection, not additional service cleanup.
"""
pass pass
@abstractmethod @abstractmethod
async def _receive_messages(self): async def _receive_messages(self):
"""Implement service-specific message receiving logic.""" """Receive and process websocket messages.
Implement service-specific logic for receiving and handling messages
from the websocket connection. Called continuously by the receive task handler.
"""
pass pass