Update the remaining docstrings, update pre-commit hook, add docstring formatting CI, update CONTRIBUTING with formatting guidance (#2089)
This commit is contained in:
6
.github/workflows/format.yaml
vendored
6
.github/workflows/format.yaml
vendored
@@ -17,7 +17,7 @@ concurrency:
|
|||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ruff-format:
|
ruff-format:
|
||||||
name: "Formatting checker"
|
name: "Code quality checks"
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repo
|
- name: Checkout repo
|
||||||
@@ -39,8 +39,8 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
source .venv/bin/activate
|
source .venv/bin/activate
|
||||||
ruff format --diff
|
ruff format --diff
|
||||||
- name: Ruff import linter
|
- name: Ruff linter (all rules)
|
||||||
id: ruff-check
|
id: ruff-check
|
||||||
run: |
|
run: |
|
||||||
source .venv/bin/activate
|
source .venv/bin/activate
|
||||||
ruff check --select I
|
ruff check
|
||||||
|
|||||||
@@ -71,6 +71,21 @@ We follow Google-style docstrings with these specific conventions:
|
|||||||
- Use `Parameters:` section to document each enum value and its meaning
|
- Use `Parameters:` section to document each enum value and its meaning
|
||||||
- No `__init__` docstring (Enums don't have custom constructors)
|
- No `__init__` docstring (Enums don't have custom constructors)
|
||||||
|
|
||||||
|
**Code Examples in Docstrings:**
|
||||||
|
|
||||||
|
- Use `Examples:` as a section header for multiple examples
|
||||||
|
- Use descriptive text followed by double colons (`::`) for each example
|
||||||
|
- **Always include a blank line after the `::"`**
|
||||||
|
- Indent all code consistently within each block
|
||||||
|
- Separate multiple examples with blank lines for readability
|
||||||
|
|
||||||
|
**Lists and Bullets in Docstrings:**
|
||||||
|
|
||||||
|
- Use dashes (`-`) for bullet points, not asterisks (`*`)
|
||||||
|
- **Add a blank line before bullet lists** when they follow a colon
|
||||||
|
- Use section headers like "Supported features:" or "Behavior:" before lists
|
||||||
|
- For complex nested information, consider using paragraph format instead
|
||||||
|
|
||||||
#### Examples:
|
#### Examples:
|
||||||
|
|
||||||
```python
|
```python
|
||||||
@@ -80,6 +95,12 @@ class MyService(BaseService):
|
|||||||
|
|
||||||
Provides detailed explanation of the service's functionality,
|
Provides detailed explanation of the service's functionality,
|
||||||
key features, and usage patterns.
|
key features, and usage patterns.
|
||||||
|
|
||||||
|
Supported features:
|
||||||
|
|
||||||
|
- Feature one with detailed explanation
|
||||||
|
- Feature two with additional context
|
||||||
|
- Feature three for advanced use cases
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, param1: str, param2: bool = True, **kwargs):
|
def __init__(self, param1: str, param2: bool = True, **kwargs):
|
||||||
@@ -127,6 +148,34 @@ class ConfigParams:
|
|||||||
port: int = 8080
|
port: int = 8080
|
||||||
timeout: float = 30.0
|
timeout: float = 30.0
|
||||||
|
|
||||||
|
# Dataclass with code examples
|
||||||
|
@dataclass
|
||||||
|
class MessageFrame:
|
||||||
|
"""Frame containing messages in OpenAI format.
|
||||||
|
|
||||||
|
Supports both simple and content list message formats.
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
Simple format::
|
||||||
|
|
||||||
|
[
|
||||||
|
{"role": "user", "content": "Hello"},
|
||||||
|
{"role": "assistant", "content": "Hi there!"}
|
||||||
|
]
|
||||||
|
|
||||||
|
Content list format::
|
||||||
|
|
||||||
|
[
|
||||||
|
{"role": "user", "content": [{"type": "text", "text": "Hello"}]},
|
||||||
|
{"role": "assistant", "content": [{"type": "text", "text": "Hi there!"}]}
|
||||||
|
]
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
messages: List of messages in OpenAI format.
|
||||||
|
"""
|
||||||
|
|
||||||
|
messages: List[dict]
|
||||||
|
|
||||||
# Enum class
|
# Enum class
|
||||||
class Status(Enum):
|
class Status(Enum):
|
||||||
"""Status codes for processing operations.
|
"""Status codes for processing operations.
|
||||||
|
|||||||
159
docs/api/conf.py
159
docs/api/conf.py
@@ -26,6 +26,10 @@ extensions = [
|
|||||||
"sphinx.ext.intersphinx",
|
"sphinx.ext.intersphinx",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
suppress_warnings = [
|
||||||
|
"autodoc.mocked_object",
|
||||||
|
]
|
||||||
|
|
||||||
# Napoleon settings
|
# Napoleon settings
|
||||||
napoleon_google_docstring = True
|
napoleon_google_docstring = True
|
||||||
napoleon_include_init_with_doc = True
|
napoleon_include_init_with_doc = True
|
||||||
@@ -71,7 +75,6 @@ autodoc_mock_imports = [
|
|||||||
"langchain",
|
"langchain",
|
||||||
"lmnt",
|
"lmnt",
|
||||||
"noisereduce",
|
"noisereduce",
|
||||||
"openai",
|
|
||||||
"openpipe",
|
"openpipe",
|
||||||
"simli",
|
"simli",
|
||||||
"soundfile",
|
"soundfile",
|
||||||
@@ -81,10 +84,6 @@ autodoc_mock_imports = [
|
|||||||
"tkinter",
|
"tkinter",
|
||||||
"daily",
|
"daily",
|
||||||
"daily_python",
|
"daily_python",
|
||||||
"pydantic.BaseModel",
|
|
||||||
"pydantic.Field",
|
|
||||||
"pydantic._internal._model_construction",
|
|
||||||
"pydantic._internal._fields",
|
|
||||||
# Moondream dependencies
|
# Moondream dependencies
|
||||||
"torch",
|
"torch",
|
||||||
"transformers",
|
"transformers",
|
||||||
@@ -167,6 +166,19 @@ autodoc_mock_imports = [
|
|||||||
"mcp.client.stdio",
|
"mcp.client.stdio",
|
||||||
"mcp.ClientSession",
|
"mcp.ClientSession",
|
||||||
"mcp.StdioServerParameters",
|
"mcp.StdioServerParameters",
|
||||||
|
# gstreamer
|
||||||
|
"gi",
|
||||||
|
"gi.require_version",
|
||||||
|
"gi.repository",
|
||||||
|
# Protobuf mocks
|
||||||
|
"pipecat.frames.protobufs.frames_pb2",
|
||||||
|
"pipecat.serializers.protobuf",
|
||||||
|
"google.protobuf",
|
||||||
|
"google.protobuf.descriptor",
|
||||||
|
"google.protobuf.descriptor_pool",
|
||||||
|
"google.protobuf.runtime_version",
|
||||||
|
"google.protobuf.symbol_database",
|
||||||
|
"google.protobuf.internal.builder",
|
||||||
]
|
]
|
||||||
|
|
||||||
# HTML output settings
|
# HTML output settings
|
||||||
@@ -176,76 +188,32 @@ autodoc_typehints = "signature" # Show type hints in the signature only, not in
|
|||||||
html_show_sphinx = False
|
html_show_sphinx = False
|
||||||
|
|
||||||
|
|
||||||
def verify_modules():
|
def import_core_modules():
|
||||||
"""Verify that required modules are available."""
|
"""Import core pipecat modules for autodoc to discover."""
|
||||||
required_modules = {
|
core_modules = [
|
||||||
"services": [
|
"pipecat",
|
||||||
"assemblyai",
|
"pipecat.frames",
|
||||||
"aws",
|
"pipecat.pipeline",
|
||||||
"cartesia",
|
"pipecat.processors",
|
||||||
"deepgram",
|
"pipecat.services",
|
||||||
"google",
|
"pipecat.transports",
|
||||||
"lmnt",
|
"pipecat.audio",
|
||||||
"riva",
|
"pipecat.adapters",
|
||||||
"simli",
|
"pipecat.clocks",
|
||||||
],
|
"pipecat.metrics",
|
||||||
"serializers": ["livekit"],
|
"pipecat.observers",
|
||||||
"vad": ["silero", "vad_analyzer"],
|
"pipecat.serializers",
|
||||||
"transports": {
|
"pipecat.sync",
|
||||||
"services": ["daily", "livekit"],
|
"pipecat.transcriptions",
|
||||||
"local": ["audio", "tk"],
|
"pipecat.utils",
|
||||||
"network": ["fastapi_websocket", "websocket_server"],
|
]
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
# Skip importing modules that are in autodoc_mock_imports
|
for module_name in core_modules:
|
||||||
skipped_modules = set(autodoc_mock_imports)
|
try:
|
||||||
|
__import__(module_name)
|
||||||
missing = []
|
logger.info(f"Successfully imported {module_name}")
|
||||||
for category, modules in required_modules.items():
|
except ImportError as e:
|
||||||
if isinstance(modules, dict):
|
logger.warning(f"Failed to import {module_name}: {e}")
|
||||||
# Handle nested structure
|
|
||||||
for subcategory, submodules in modules.items():
|
|
||||||
for module in submodules:
|
|
||||||
# Check if module is in autodoc_mock_imports
|
|
||||||
if (
|
|
||||||
f"pipecat.{category}.{subcategory}.{module}" in skipped_modules
|
|
||||||
or module in skipped_modules
|
|
||||||
):
|
|
||||||
logger.info(
|
|
||||||
f"Skipping import of mocked module: pipecat.{category}.{subcategory}.{module}"
|
|
||||||
)
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
__import__(f"pipecat.{category}.{subcategory}.{module}")
|
|
||||||
logger.info(
|
|
||||||
f"Successfully imported pipecat.{category}.{subcategory}.{module}"
|
|
||||||
)
|
|
||||||
except (ImportError, TypeError, NameError) as e:
|
|
||||||
missing.append(f"pipecat.{category}.{subcategory}.{module}")
|
|
||||||
logger.warning(
|
|
||||||
f"Optional module not available: pipecat.{category}.{subcategory}.{module} - {str(e)}"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# Handle flat structure
|
|
||||||
for module in modules:
|
|
||||||
# Check if module is in autodoc_mock_imports
|
|
||||||
if f"pipecat.{category}.{module}" in skipped_modules or module in skipped_modules:
|
|
||||||
logger.info(f"Skipping import of mocked module: pipecat.{category}.{module}")
|
|
||||||
continue
|
|
||||||
|
|
||||||
try:
|
|
||||||
__import__(f"pipecat.{category}.{module}")
|
|
||||||
logger.info(f"Successfully imported pipecat.{category}.{module}")
|
|
||||||
except (ImportError, TypeError, NameError) as e:
|
|
||||||
missing.append(f"pipecat.{category}.{module}")
|
|
||||||
logger.warning(
|
|
||||||
f"Optional module not available: pipecat.{category}.{module} - {str(e)}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if missing:
|
|
||||||
logger.warning(f"Some optional modules are not available: {missing}")
|
|
||||||
|
|
||||||
|
|
||||||
def clean_title(title: str) -> str:
|
def clean_title(title: str) -> str:
|
||||||
@@ -257,40 +225,7 @@ def clean_title(title: str) -> str:
|
|||||||
parts = title.split(".")
|
parts = title.split(".")
|
||||||
title = parts[-1]
|
title = parts[-1]
|
||||||
|
|
||||||
# Special cases for service names and common acronyms
|
return title
|
||||||
special_cases = {
|
|
||||||
"ai": "AI",
|
|
||||||
"aws": "AWS",
|
|
||||||
"api": "API",
|
|
||||||
"vad": "VAD",
|
|
||||||
"assemblyai": "AssemblyAI",
|
|
||||||
"deepgram": "Deepgram",
|
|
||||||
"elevenlabs": "ElevenLabs",
|
|
||||||
"openai": "OpenAI",
|
|
||||||
"openpipe": "OpenPipe",
|
|
||||||
"playht": "PlayHT",
|
|
||||||
"xtts": "XTTS",
|
|
||||||
"lmnt": "LMNT",
|
|
||||||
"stt": "STT",
|
|
||||||
"tts": "TTS",
|
|
||||||
"llm": "LLM",
|
|
||||||
"rtvi": "RTVI",
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check if the entire title is a special case
|
|
||||||
if title.lower() in special_cases:
|
|
||||||
return special_cases[title.lower()]
|
|
||||||
|
|
||||||
# Otherwise, capitalize each word
|
|
||||||
words = title.split("_")
|
|
||||||
cleaned_words = []
|
|
||||||
for word in words:
|
|
||||||
if word.lower() in special_cases:
|
|
||||||
cleaned_words.append(special_cases[word.lower()])
|
|
||||||
else:
|
|
||||||
cleaned_words.append(word.capitalize())
|
|
||||||
|
|
||||||
return " ".join(cleaned_words)
|
|
||||||
|
|
||||||
|
|
||||||
def setup(app):
|
def setup(app):
|
||||||
@@ -315,9 +250,8 @@ def setup(app):
|
|||||||
|
|
||||||
excludes = [
|
excludes = [
|
||||||
str(project_root / "src/pipecat/pipeline/to_be_updated"),
|
str(project_root / "src/pipecat/pipeline/to_be_updated"),
|
||||||
str(project_root / "src/pipecat/processors/gstreamer"),
|
str(project_root / "src/pipecat/examples"),
|
||||||
str(project_root / "src/pipecat/services/to_be_updated"),
|
str(project_root / "src/pipecat/tests"),
|
||||||
str(project_root / "src/pipecat/vad"), # deprecated
|
|
||||||
"**/test_*.py",
|
"**/test_*.py",
|
||||||
"**/tests/*.py",
|
"**/tests/*.py",
|
||||||
]
|
]
|
||||||
@@ -358,5 +292,4 @@ def setup(app):
|
|||||||
logger.error(f"Error generating API documentation: {e}", exc_info=True)
|
logger.error(f"Error generating API documentation: {e}", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
# Run module verification
|
import_core_modules()
|
||||||
verify_modules()
|
|
||||||
|
|||||||
@@ -1,57 +1,17 @@
|
|||||||
Pipecat API Reference Docs
|
Pipecat API Reference
|
||||||
==========================
|
=====================
|
||||||
|
|
||||||
Welcome to Pipecat's API reference documentation!
|
Welcome to the Pipecat API reference.
|
||||||
|
|
||||||
Pipecat is an open source framework for building voice and multimodal assistants.
|
Use the navigation on the left to browse modules, or search using the search box.
|
||||||
It provides a flexible pipeline architecture for connecting various AI services,
|
|
||||||
audio processing, and transport layers.
|
**New to Pipecat?** Check out the `main documentation <https://docs.pipecat.ai>`_ for tutorials, guides, and client SDK information.
|
||||||
|
|
||||||
Quick Links
|
Quick Links
|
||||||
-----------
|
-----------
|
||||||
|
|
||||||
* `GitHub Repository <https://github.com/pipecat-ai/pipecat>`_
|
* `GitHub Repository <https://github.com/pipecat-ai/pipecat>`_
|
||||||
* `Website <https://pipecat.ai>`_
|
* `Join our Community <https://discord.gg/pipecat>`_
|
||||||
|
|
||||||
API Reference
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Core Components
|
|
||||||
~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
* :mod:`Frames <pipecat.frames>`
|
|
||||||
* :mod:`Processors <pipecat.processors>`
|
|
||||||
* :mod:`Pipeline <pipecat.pipeline>`
|
|
||||||
|
|
||||||
Audio Processing
|
|
||||||
~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
* :mod:`Audio <pipecat.audio>`
|
|
||||||
|
|
||||||
Services
|
|
||||||
~~~~~~~~
|
|
||||||
|
|
||||||
* :mod:`Services <pipecat.services>`
|
|
||||||
|
|
||||||
Transport & Serialization
|
|
||||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
|
||||||
|
|
||||||
* :mod:`Transports <pipecat.transports>`
|
|
||||||
* :mod:`Local <pipecat.transports.local>`
|
|
||||||
* :mod:`Network <pipecat.transports.network>`
|
|
||||||
* :mod:`Services <pipecat.transports.services>`
|
|
||||||
* :mod:`Serializers <pipecat.serializers>`
|
|
||||||
|
|
||||||
Utilities
|
|
||||||
~~~~~~~~~
|
|
||||||
|
|
||||||
* :mod:`Adapters <pipecat.adapters>`
|
|
||||||
* :mod:`Clocks <pipecat.clocks>`
|
|
||||||
* :mod:`Metrics <pipecat.metrics>`
|
|
||||||
* :mod:`Observers <pipecat.observers>`
|
|
||||||
* :mod:`Sync <pipecat.sync>`
|
|
||||||
* :mod:`Transcriptions <pipecat.transcriptions>`
|
|
||||||
* :mod:`Utils <pipecat.utils>`
|
|
||||||
|
|
||||||
.. toctree::
|
.. toctree::
|
||||||
:maxdepth: 3
|
:maxdepth: 3
|
||||||
@@ -72,10 +32,3 @@ Utilities
|
|||||||
Transcriptions <api/pipecat.transcriptions>
|
Transcriptions <api/pipecat.transcriptions>
|
||||||
Transports <api/pipecat.transports>
|
Transports <api/pipecat.transports>
|
||||||
Utils <api/pipecat.utils>
|
Utils <api/pipecat.utils>
|
||||||
|
|
||||||
Indices and tables
|
|
||||||
==================
|
|
||||||
|
|
||||||
* :ref:`genindex`
|
|
||||||
* :ref:`modindex`
|
|
||||||
* :ref:`search`
|
|
||||||
@@ -123,9 +123,21 @@ select = [
|
|||||||
"D", # Docstring rules
|
"D", # Docstring rules
|
||||||
"I", # Import rules
|
"I", # Import rules
|
||||||
]
|
]
|
||||||
|
ignore = [
|
||||||
|
"D105", # Missing docstring in magic methods (__str__, __repr__, etc.)
|
||||||
|
]
|
||||||
|
|
||||||
[tool.ruff.lint.per-file-ignores]
|
[tool.ruff.lint.per-file-ignores]
|
||||||
|
# Skip docstring checks for non-source code
|
||||||
|
"examples/**/*.py" = ["D"]
|
||||||
|
"tests/**/*.py" = ["D"]
|
||||||
|
"scripts/**/*.py" = ["D"]
|
||||||
|
"docs/**/*.py" = ["D"]
|
||||||
|
# Skip D104 (missing docstring in public package) for __init__.py files
|
||||||
"**/__init__.py" = ["D104"]
|
"**/__init__.py" = ["D104"]
|
||||||
|
# Skip specific rules for generated protobuf files
|
||||||
|
"**/*_pb2.py" = ["D"]
|
||||||
|
"src/pipecat/services/__init__.py" = ["D"]
|
||||||
|
|
||||||
[tool.ruff.lint.pydocstyle]
|
[tool.ruff.lint.pydocstyle]
|
||||||
convention = "google"
|
convention = "google"
|
||||||
|
|||||||
@@ -1,3 +1,27 @@
|
|||||||
#!/bin/sh
|
#!/bin/bash
|
||||||
|
|
||||||
NO_COLOR=1 ruff format --diff
|
# Color codes for output
|
||||||
|
RED='\033[0;31m'
|
||||||
|
GREEN='\033[0;32m'
|
||||||
|
NC='\033[0m' # No Color
|
||||||
|
|
||||||
|
echo "🔍 Running pre-commit checks..."
|
||||||
|
|
||||||
|
# Change to project root (one level up from scripts/)
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
|
||||||
|
# Format check
|
||||||
|
echo "📝 Checking code formatting..."
|
||||||
|
if ! NO_COLOR=1 ruff format --diff --check; then
|
||||||
|
echo -e "${RED}❌ Code formatting issues found. Run 'ruff format' to fix.${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Lint check
|
||||||
|
echo "🔍 Running linter..."
|
||||||
|
if ! ruff check; then
|
||||||
|
echo -e "${RED}❌ Linting issues found.${NC}"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo -e "${GREEN}✅ All pre-commit checks passed!${NC}"
|
||||||
@@ -1,3 +1,15 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Base adapter for LLM provider integration.
|
||||||
|
|
||||||
|
This module provides the abstract base class for implementing LLM provider-specific
|
||||||
|
adapters that handle tool format conversion and standardization.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from typing import Any, List, Union, cast
|
from typing import Any, List, Union, cast
|
||||||
|
|
||||||
@@ -7,12 +19,35 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|||||||
|
|
||||||
|
|
||||||
class BaseLLMAdapter(ABC):
|
class BaseLLMAdapter(ABC):
|
||||||
|
"""Abstract base class for LLM provider adapters.
|
||||||
|
|
||||||
|
Provides a standard interface for converting between Pipecat's standardized
|
||||||
|
tool schemas and provider-specific tool formats. Subclasses must implement
|
||||||
|
provider-specific conversion logic.
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Any]:
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Any]:
|
||||||
"""Converts tools to the provider's format."""
|
"""Convert tools schema to the provider's specific format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
tools_schema: The standardized tools schema to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of tools in the provider's expected format.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def from_standard_tools(self, tools: Any) -> List[Any]:
|
def from_standard_tools(self, tools: Any) -> List[Any]:
|
||||||
|
"""Convert tools from standard format to provider format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
tools: Tools in standard format or provider-specific format.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of tools converted to provider format, or original tools
|
||||||
|
if not in standard format.
|
||||||
|
"""
|
||||||
if isinstance(tools, ToolsSchema):
|
if isinstance(tools, ToolsSchema):
|
||||||
logger.debug(f"Retrieving the tools using the adapter: {type(self)}")
|
logger.debug(f"Retrieving the tools using the adapter: {type(self)}")
|
||||||
return self.to_provider_tools_format(tools)
|
return self.to_provider_tools_format(tools)
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Function schema utilities for AI tool definitions.
|
||||||
|
|
||||||
|
This module provides standardized function schema representation for defining
|
||||||
|
tools and functions used with AI models, ensuring consistent formatting
|
||||||
|
across different AI service providers.
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
|
|
||||||
@@ -13,17 +20,19 @@ class FunctionSchema:
|
|||||||
Provides a structured way to define function tools used with AI models like OpenAI.
|
Provides a structured way to define function tools used with AI models like OpenAI.
|
||||||
This schema defines the function's name, description, parameter properties, and
|
This schema defines the function's name, description, parameter properties, and
|
||||||
required parameters, following specifications required by AI service providers.
|
required parameters, following specifications required by AI service providers.
|
||||||
|
|
||||||
Args:
|
|
||||||
name: Name of the function to be called.
|
|
||||||
description: Description of what the function does.
|
|
||||||
properties: Dictionary defining parameter types, descriptions, and constraints.
|
|
||||||
required: List of property names that are required parameters.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self, name: str, description: str, properties: Dict[str, Any], required: List[str]
|
self, name: str, description: str, properties: Dict[str, Any], required: List[str]
|
||||||
) -> None:
|
) -> None:
|
||||||
|
"""Initialize the function schema.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: Name of the function to be called.
|
||||||
|
description: Description of what the function does.
|
||||||
|
properties: Dictionary defining parameter types, descriptions, and constraints.
|
||||||
|
required: List of property names that are required parameters.
|
||||||
|
"""
|
||||||
self._name = name
|
self._name = name
|
||||||
self._description = description
|
self._description = description
|
||||||
self._properties = properties
|
self._properties = properties
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Tools schema definitions for function calling adapters.
|
||||||
|
|
||||||
|
This module provides schemas for managing both standardized function tools
|
||||||
|
and custom adapter-specific tools in the Pipecat framework.
|
||||||
|
"""
|
||||||
|
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional
|
||||||
|
|
||||||
@@ -11,33 +17,61 @@ from pipecat.adapters.schemas.function_schema import FunctionSchema
|
|||||||
|
|
||||||
|
|
||||||
class AdapterType(Enum):
|
class AdapterType(Enum):
|
||||||
|
"""Supported adapter types for custom tools.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
GEMINI: Google Gemini adapter - currently the only service supporting custom tools.
|
||||||
|
"""
|
||||||
|
|
||||||
GEMINI = "gemini" # that is the only service where we are able to add custom tools for now
|
GEMINI = "gemini" # that is the only service where we are able to add custom tools for now
|
||||||
|
|
||||||
|
|
||||||
class ToolsSchema:
|
class ToolsSchema:
|
||||||
|
"""Schema for managing both standard and custom function calling tools.
|
||||||
|
|
||||||
|
This class provides a unified interface for handling standardized function
|
||||||
|
schemas alongside custom tools that may not follow the standard format,
|
||||||
|
such as adapter-specific search tools.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
standard_tools: List[FunctionSchema],
|
standard_tools: List[FunctionSchema],
|
||||||
custom_tools: Optional[Dict[AdapterType, List[Dict[str, Any]]]] = None,
|
custom_tools: Optional[Dict[AdapterType, List[Dict[str, Any]]]] = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
"""
|
"""Initialize the tools schema.
|
||||||
A schema for tools that includes both standardized function schemas
|
|
||||||
and custom tools that do not follow the FunctionSchema format.
|
|
||||||
|
|
||||||
:param standard_tools: List of tools following FunctionSchema.
|
Args:
|
||||||
:param custom_tools: List of tools in a custom format (e.g., search_tool).
|
standard_tools: List of tools following the standardized FunctionSchema format.
|
||||||
|
custom_tools: Dictionary mapping adapter types to their custom tool definitions.
|
||||||
|
These tools may not follow the FunctionSchema format (e.g., search_tool).
|
||||||
"""
|
"""
|
||||||
self._standard_tools = standard_tools
|
self._standard_tools = standard_tools
|
||||||
self._custom_tools = custom_tools
|
self._custom_tools = custom_tools
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def standard_tools(self) -> List[FunctionSchema]:
|
def standard_tools(self) -> List[FunctionSchema]:
|
||||||
|
"""Get the list of standard function schema tools.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of tools following the FunctionSchema format.
|
||||||
|
"""
|
||||||
return self._standard_tools
|
return self._standard_tools
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def custom_tools(self) -> Dict[AdapterType, List[Dict[str, Any]]]:
|
def custom_tools(self) -> Dict[AdapterType, List[Dict[str, Any]]]:
|
||||||
|
"""Get the custom tools dictionary.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary mapping adapter types to their custom tool definitions.
|
||||||
|
"""
|
||||||
return self._custom_tools
|
return self._custom_tools
|
||||||
|
|
||||||
@custom_tools.setter
|
@custom_tools.setter
|
||||||
def custom_tools(self, value: Dict[AdapterType, List[Dict[str, Any]]]) -> None:
|
def custom_tools(self, value: Dict[AdapterType, List[Dict[str, Any]]]) -> None:
|
||||||
|
"""Set the custom tools dictionary.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
value: Dictionary mapping adapter types to their custom tool definitions.
|
||||||
|
"""
|
||||||
self._custom_tools = value
|
self._custom_tools = value
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Anthropic LLM adapter for Pipecat."""
|
||||||
|
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
||||||
@@ -12,8 +14,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|||||||
|
|
||||||
|
|
||||||
class AnthropicLLMAdapter(BaseLLMAdapter):
|
class AnthropicLLMAdapter(BaseLLMAdapter):
|
||||||
|
"""Adapter for converting tool schemas to Anthropic's function-calling format.
|
||||||
|
|
||||||
|
This adapter handles the conversion of Pipecat's standard function schemas
|
||||||
|
to the specific format required by Anthropic's Claude models for function calling.
|
||||||
|
"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_anthropic_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
def _to_anthropic_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
||||||
|
"""Convert a single function schema to Anthropic's format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
function: The function schema to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary containing the function definition in Anthropic's format.
|
||||||
|
"""
|
||||||
return {
|
return {
|
||||||
"name": function.name,
|
"name": function.name,
|
||||||
"description": function.description,
|
"description": function.description,
|
||||||
@@ -25,10 +41,13 @@ class AnthropicLLMAdapter(BaseLLMAdapter):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
||||||
"""Converts function schemas to Anthropic's function-calling format.
|
"""Convert function schemas to Anthropic's function-calling format.
|
||||||
|
|
||||||
:return: Anthropic formatted function call definition.
|
Args:
|
||||||
|
tools_schema: The tools schema containing functions to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of function definitions formatted for Anthropic's API.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
functions_schema = tools_schema.standard_tools
|
functions_schema = tools_schema.standard_tools
|
||||||
return [self._to_anthropic_function_format(func) for func in functions_schema]
|
return [self._to_anthropic_function_format(func) for func in functions_schema]
|
||||||
|
|||||||
@@ -3,6 +3,9 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""AWS Nova Sonic LLM adapter for Pipecat."""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
@@ -12,8 +15,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|||||||
|
|
||||||
|
|
||||||
class AWSNovaSonicLLMAdapter(BaseLLMAdapter):
|
class AWSNovaSonicLLMAdapter(BaseLLMAdapter):
|
||||||
|
"""Adapter for AWS Nova Sonic language models.
|
||||||
|
|
||||||
|
Converts Pipecat's standard function schemas into AWS Nova Sonic's
|
||||||
|
specific function-calling format, enabling tool use with Nova Sonic models.
|
||||||
|
"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_aws_nova_sonic_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
def _to_aws_nova_sonic_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
||||||
|
"""Convert a function schema to AWS Nova Sonic format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
function: The function schema to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary in AWS Nova Sonic function format with toolSpec structure.
|
||||||
|
"""
|
||||||
return {
|
return {
|
||||||
"toolSpec": {
|
"toolSpec": {
|
||||||
"name": function.name,
|
"name": function.name,
|
||||||
@@ -31,10 +48,13 @@ class AWSNovaSonicLLMAdapter(BaseLLMAdapter):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
||||||
"""Converts function schemas to AWS Nova Sonic function-calling format.
|
"""Convert tools schema to AWS Nova Sonic function-calling format.
|
||||||
|
|
||||||
:return: AWS Nova Sonic formatted function call definition.
|
Args:
|
||||||
|
tools_schema: The tools schema containing function definitions to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of dictionaries in AWS Nova Sonic function format.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
functions_schema = tools_schema.standard_tools
|
functions_schema = tools_schema.standard_tools
|
||||||
return [self._to_aws_nova_sonic_function_format(func) for func in functions_schema]
|
return [self._to_aws_nova_sonic_function_format(func) for func in functions_schema]
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""AWS Bedrock LLM adapter for Pipecat."""
|
||||||
|
|
||||||
from typing import Any, Dict, List
|
from typing import Any, Dict, List
|
||||||
|
|
||||||
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
||||||
@@ -12,8 +14,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|||||||
|
|
||||||
|
|
||||||
class AWSBedrockLLMAdapter(BaseLLMAdapter):
|
class AWSBedrockLLMAdapter(BaseLLMAdapter):
|
||||||
|
"""Adapter for AWS Bedrock LLM integration with Pipecat.
|
||||||
|
|
||||||
|
Provides conversion utilities for transforming Pipecat function schemas
|
||||||
|
into AWS Bedrock's expected tool format for function calling capabilities.
|
||||||
|
"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_bedrock_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
def _to_bedrock_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
||||||
|
"""Convert a function schema to Bedrock's tool format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
function: The function schema to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary formatted for Bedrock's tool specification.
|
||||||
|
"""
|
||||||
return {
|
return {
|
||||||
"toolSpec": {
|
"toolSpec": {
|
||||||
"name": function.name,
|
"name": function.name,
|
||||||
@@ -29,10 +45,13 @@ class AWSBedrockLLMAdapter(BaseLLMAdapter):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
||||||
"""Converts function schemas to Bedrock's function-calling format.
|
"""Convert function schemas to Bedrock's function-calling format.
|
||||||
|
|
||||||
:return: Bedrock formatted function call definition.
|
Args:
|
||||||
|
tools_schema: The tools schema containing functions to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of Bedrock formatted function call definitions.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
functions_schema = tools_schema.standard_tools
|
functions_schema = tools_schema.standard_tools
|
||||||
return [self._to_bedrock_function_format(func) for func in functions_schema]
|
return [self._to_bedrock_function_format(func) for func in functions_schema]
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Gemini LLM adapter for Pipecat."""
|
||||||
|
|
||||||
from typing import Any, Dict, List, Union
|
from typing import Any, Dict, List, Union
|
||||||
|
|
||||||
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
||||||
@@ -11,12 +13,23 @@ from pipecat.adapters.schemas.tools_schema import AdapterType, ToolsSchema
|
|||||||
|
|
||||||
|
|
||||||
class GeminiLLMAdapter(BaseLLMAdapter):
|
class GeminiLLMAdapter(BaseLLMAdapter):
|
||||||
|
"""LLM adapter for Google's Gemini service.
|
||||||
|
|
||||||
|
Provides tool schema conversion functionality to transform standard tool
|
||||||
|
definitions into Gemini's specific function-calling format for use with
|
||||||
|
Gemini LLM models.
|
||||||
|
"""
|
||||||
|
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
||||||
"""Converts function schemas to Gemini's function-calling format.
|
"""Convert tool schemas to Gemini's function-calling format.
|
||||||
|
|
||||||
:return: Gemini formatted function call definition.
|
Args:
|
||||||
|
tools_schema: The tools schema containing standard and custom tool definitions.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of tool definitions formatted for Gemini's function-calling API.
|
||||||
|
Includes both converted standard tools and any custom Gemini-specific tools.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
functions_schema = tools_schema.standard_tools
|
functions_schema = tools_schema.standard_tools
|
||||||
formatted_standard_tools = [
|
formatted_standard_tools = [
|
||||||
{"function_declarations": [func.to_default_dict() for func in functions_schema]}
|
{"function_declarations": [func.to_default_dict() for func in functions_schema]}
|
||||||
|
|||||||
@@ -3,6 +3,9 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""OpenAI LLM adapter for Pipecat."""
|
||||||
|
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
from openai.types.chat import ChatCompletionToolParam
|
from openai.types.chat import ChatCompletionToolParam
|
||||||
@@ -12,10 +15,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|||||||
|
|
||||||
|
|
||||||
class OpenAILLMAdapter(BaseLLMAdapter):
|
class OpenAILLMAdapter(BaseLLMAdapter):
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ChatCompletionToolParam]:
|
"""Adapter for converting tool schemas to OpenAI's format.
|
||||||
"""Converts function schemas to OpenAI's function-calling format.
|
|
||||||
|
|
||||||
:return: OpenAI formatted function call definition.
|
Provides conversion utilities for transforming Pipecat's standard tool
|
||||||
|
schemas into the format expected by OpenAI's ChatCompletion API for
|
||||||
|
function calling capabilities.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[ChatCompletionToolParam]:
|
||||||
|
"""Convert function schemas to OpenAI's function-calling format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
tools_schema: The Pipecat tools schema to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of OpenAI formatted function call definitions ready for use
|
||||||
|
with ChatCompletion API.
|
||||||
"""
|
"""
|
||||||
functions_schema = tools_schema.standard_tools
|
functions_schema = tools_schema.standard_tools
|
||||||
return [
|
return [
|
||||||
|
|||||||
@@ -3,6 +3,9 @@
|
|||||||
#
|
#
|
||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""OpenAI Realtime LLM adapter for Pipecat."""
|
||||||
|
|
||||||
from typing import Any, Dict, List, Union
|
from typing import Any, Dict, List, Union
|
||||||
|
|
||||||
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
from pipecat.adapters.base_llm_adapter import BaseLLMAdapter
|
||||||
@@ -11,8 +14,22 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
|||||||
|
|
||||||
|
|
||||||
class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
||||||
|
"""LLM adapter for OpenAI Realtime API function calling.
|
||||||
|
|
||||||
|
Converts Pipecat's tool schemas into the specific format required by
|
||||||
|
OpenAI's Realtime API for function calling capabilities.
|
||||||
|
"""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _to_openai_realtime_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
def _to_openai_realtime_function_format(function: FunctionSchema) -> Dict[str, Any]:
|
||||||
|
"""Convert a function schema to OpenAI Realtime format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
function: The function schema to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary in OpenAI Realtime function format.
|
||||||
|
"""
|
||||||
return {
|
return {
|
||||||
"type": "function",
|
"type": "function",
|
||||||
"name": function.name,
|
"name": function.name,
|
||||||
@@ -25,10 +42,13 @@ class OpenAIRealtimeLLMAdapter(BaseLLMAdapter):
|
|||||||
}
|
}
|
||||||
|
|
||||||
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
def to_provider_tools_format(self, tools_schema: ToolsSchema) -> List[Dict[str, Any]]:
|
||||||
"""Converts function schemas to Openai Realtime function-calling format.
|
"""Convert tool schemas to OpenAI Realtime function-calling format.
|
||||||
|
|
||||||
:return: Openai Realtime formatted function call definition.
|
Args:
|
||||||
|
tools_schema: The tools schema containing functions to convert.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of function definitions in OpenAI Realtime format.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
functions_schema = tools_schema.standard_tools
|
functions_schema = tools_schema.standard_tools
|
||||||
return [self._to_openai_realtime_function_format(func) for func in functions_schema]
|
return [self._to_openai_realtime_function_format(func) for func in functions_schema]
|
||||||
|
|||||||
@@ -4,44 +4,68 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base audio filter interface for input transport audio processing.
|
||||||
|
|
||||||
|
This module provides the abstract base class for implementing audio filters
|
||||||
|
that process audio data before VAD and downstream processing in input transports.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
from pipecat.frames.frames import FilterControlFrame
|
from pipecat.frames.frames import FilterControlFrame
|
||||||
|
|
||||||
|
|
||||||
class BaseAudioFilter(ABC):
|
class BaseAudioFilter(ABC):
|
||||||
"""This is a base class for input transport audio filters. If an audio
|
"""Base class for input transport audio filters.
|
||||||
|
|
||||||
|
This is a base class for input transport audio filters. If an audio
|
||||||
filter is provided to the input transport it will be used to process audio
|
filter is provided to the input transport it will be used to process audio
|
||||||
before VAD and before pushing it downstream. There are control frames to
|
before VAD and before pushing it downstream. There are control frames to
|
||||||
update filter settings or to enable or disable the filter at runtime.
|
update filter settings or to enable or disable the filter at runtime.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def start(self, sample_rate: int):
|
async def start(self, sample_rate: int):
|
||||||
"""This will be called from the input transport when the transport is
|
"""Initialize the filter when the input transport starts.
|
||||||
|
|
||||||
|
This will be called from the input transport when the transport is
|
||||||
started. It can be used to initialize the filter. The input transport
|
started. It can be used to initialize the filter. The input transport
|
||||||
sample rate is provided so the filter can adjust to that sample rate.
|
sample rate is provided so the filter can adjust to that sample rate.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: The sample rate of the input transport in Hz.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
"""This will be called from the input transport when the transport is
|
"""Clean up the filter when the input transport stops.
|
||||||
stopping.
|
|
||||||
|
|
||||||
|
This will be called from the input transport when the transport is
|
||||||
|
stopping.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def process_frame(self, frame: FilterControlFrame):
|
async def process_frame(self, frame: FilterControlFrame):
|
||||||
"""This will be called when the input transport receives a
|
"""Process control frames for runtime filter configuration.
|
||||||
|
|
||||||
|
This will be called when the input transport receives a
|
||||||
FilterControlFrame.
|
FilterControlFrame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The control frame containing filter commands or settings.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def filter(self, audio: bytes) -> bytes:
|
async def filter(self, audio: bytes) -> bytes:
|
||||||
|
"""Apply the audio filter to the provided audio data.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio data as bytes to be filtered.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Filtered audio data as bytes.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Koala noise suppression audio filter for Pipecat.
|
||||||
|
|
||||||
|
This module provides an audio filter implementation using PicoVoice's Koala
|
||||||
|
Noise Suppression engine to reduce background noise in audio streams.
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import Sequence
|
from typing import Sequence
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -21,12 +27,19 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class KoalaFilter(BaseAudioFilter):
|
class KoalaFilter(BaseAudioFilter):
|
||||||
"""This is an audio filter that uses Koala Noise Suppression (from
|
"""Audio filter using Koala Noise Suppression from PicoVoice.
|
||||||
PicoVoice).
|
|
||||||
|
|
||||||
|
Provides real-time noise suppression for audio streams using PicoVoice's
|
||||||
|
Koala engine. The filter buffers audio data to match Koala's required
|
||||||
|
frame length and processes it in chunks.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, *, access_key: str) -> None:
|
def __init__(self, *, access_key: str) -> None:
|
||||||
|
"""Initialize the Koala noise suppression filter.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
access_key: PicoVoice access key for Koala engine authentication.
|
||||||
|
"""
|
||||||
self._access_key = access_key
|
self._access_key = access_key
|
||||||
|
|
||||||
self._filtering = True
|
self._filtering = True
|
||||||
@@ -36,6 +49,11 @@ class KoalaFilter(BaseAudioFilter):
|
|||||||
self._audio_buffer = bytearray()
|
self._audio_buffer = bytearray()
|
||||||
|
|
||||||
async def start(self, sample_rate: int):
|
async def start(self, sample_rate: int):
|
||||||
|
"""Initialize the filter with the transport's sample rate.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: The sample rate of the input transport in Hz.
|
||||||
|
"""
|
||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
if self._sample_rate != self._koala.sample_rate:
|
if self._sample_rate != self._koala.sample_rate:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -44,13 +62,30 @@ class KoalaFilter(BaseAudioFilter):
|
|||||||
self._koala_ready = False
|
self._koala_ready = False
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
|
"""Clean up the Koala engine when stopping."""
|
||||||
self._koala.reset()
|
self._koala.reset()
|
||||||
|
|
||||||
async def process_frame(self, frame: FilterControlFrame):
|
async def process_frame(self, frame: FilterControlFrame):
|
||||||
|
"""Process control frames to enable/disable filtering.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The control frame containing filter commands.
|
||||||
|
"""
|
||||||
if isinstance(frame, FilterEnableFrame):
|
if isinstance(frame, FilterEnableFrame):
|
||||||
self._filtering = frame.enable
|
self._filtering = frame.enable
|
||||||
|
|
||||||
async def filter(self, audio: bytes) -> bytes:
|
async def filter(self, audio: bytes) -> bytes:
|
||||||
|
"""Apply Koala noise suppression to audio data.
|
||||||
|
|
||||||
|
Buffers incoming audio and processes it in chunks that match Koala's
|
||||||
|
required frame length. Returns filtered audio data.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio data as bytes to be filtered.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Noise-suppressed audio data as bytes.
|
||||||
|
"""
|
||||||
if not self._koala_ready or not self._filtering:
|
if not self._koala_ready or not self._filtering:
|
||||||
return audio
|
return audio
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Krisp noise reduction audio filter for Pipecat.
|
||||||
|
|
||||||
|
This module provides an audio filter implementation using Krisp's noise
|
||||||
|
reduction technology to suppress background noise in audio streams.
|
||||||
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -21,14 +27,27 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class KrispProcessorManager:
|
class KrispProcessorManager:
|
||||||
"""
|
"""Singleton manager for KrispAudioProcessor instances.
|
||||||
Ensures that only one KrispAudioProcessor instance exists for the entire program.
|
|
||||||
|
Ensures that only one KrispAudioProcessor instance exists for the entire
|
||||||
|
program.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
_krisp_instance = None
|
_krisp_instance = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def get_processor(cls, sample_rate: int, sample_type: str, channels: int, model_path: str):
|
def get_processor(cls, sample_rate: int, sample_type: str, channels: int, model_path: str):
|
||||||
|
"""Get or create a KrispAudioProcessor instance.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: Audio sample rate in Hz.
|
||||||
|
sample_type: Audio sample type (e.g., "PCM_16").
|
||||||
|
channels: Number of audio channels.
|
||||||
|
model_path: Path to the Krisp model file.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Shared KrispAudioProcessor instance.
|
||||||
|
"""
|
||||||
if cls._krisp_instance is None:
|
if cls._krisp_instance is None:
|
||||||
cls._krisp_instance = KrispAudioProcessor(
|
cls._krisp_instance = KrispAudioProcessor(
|
||||||
sample_rate, sample_type, channels, model_path
|
sample_rate, sample_type, channels, model_path
|
||||||
@@ -37,14 +56,26 @@ class KrispProcessorManager:
|
|||||||
|
|
||||||
|
|
||||||
class KrispFilter(BaseAudioFilter):
|
class KrispFilter(BaseAudioFilter):
|
||||||
|
"""Audio filter using Krisp noise reduction technology.
|
||||||
|
|
||||||
|
Provides real-time noise reduction for audio streams using Krisp's
|
||||||
|
proprietary noise suppression algorithms. Requires a Krisp model file
|
||||||
|
for operation.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None
|
self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Initializes the KrispAudioProcessor with customizable audio processing settings.
|
"""Initialize the Krisp noise reduction filter.
|
||||||
|
|
||||||
:param sample_type: The type of audio sample, default is 'PCM_16'.
|
Args:
|
||||||
:param channels: Number of audio channels, default is 1.
|
sample_type: The audio sample format. Defaults to "PCM_16".
|
||||||
:param model_path: Path to the Krisp model; defaults to environment variable KRISP_MODEL_PATH if not provided.
|
channels: Number of audio channels. Defaults to 1.
|
||||||
|
model_path: Path to the Krisp model file. If None, uses KRISP_MODEL_PATH
|
||||||
|
environment variable.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If model_path is not provided and KRISP_MODEL_PATH is not set.
|
||||||
"""
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@@ -63,19 +94,41 @@ class KrispFilter(BaseAudioFilter):
|
|||||||
self._krisp_processor = None
|
self._krisp_processor = None
|
||||||
|
|
||||||
async def start(self, sample_rate: int):
|
async def start(self, sample_rate: int):
|
||||||
|
"""Initialize the Krisp processor with the transport's sample rate.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: The sample rate of the input transport in Hz.
|
||||||
|
"""
|
||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
self._krisp_processor = KrispProcessorManager.get_processor(
|
self._krisp_processor = KrispProcessorManager.get_processor(
|
||||||
self._sample_rate, self._sample_type, self._channels, self._model_path
|
self._sample_rate, self._sample_type, self._channels, self._model_path
|
||||||
)
|
)
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
|
"""Clean up the Krisp processor when stopping."""
|
||||||
self._krisp_processor = None
|
self._krisp_processor = None
|
||||||
|
|
||||||
async def process_frame(self, frame: FilterControlFrame):
|
async def process_frame(self, frame: FilterControlFrame):
|
||||||
|
"""Process control frames to enable/disable filtering.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The control frame containing filter commands.
|
||||||
|
"""
|
||||||
if isinstance(frame, FilterEnableFrame):
|
if isinstance(frame, FilterEnableFrame):
|
||||||
self._filtering = frame.enable
|
self._filtering = frame.enable
|
||||||
|
|
||||||
async def filter(self, audio: bytes) -> bytes:
|
async def filter(self, audio: bytes) -> bytes:
|
||||||
|
"""Apply Krisp noise reduction to audio data.
|
||||||
|
|
||||||
|
Converts audio to float32, applies Krisp noise reduction processing,
|
||||||
|
and returns the filtered audio clipped to int16 range.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio data as bytes to be filtered.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Noise-reduced audio data as bytes.
|
||||||
|
"""
|
||||||
if not self._filtering:
|
if not self._filtering:
|
||||||
return audio
|
return audio
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Noisereduce audio filter for Pipecat.
|
||||||
|
|
||||||
|
This module provides an audio filter implementation using the noisereduce
|
||||||
|
library to reduce background noise in audio streams through spectral
|
||||||
|
gating algorithms.
|
||||||
|
"""
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
@@ -21,21 +28,51 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class NoisereduceFilter(BaseAudioFilter):
|
class NoisereduceFilter(BaseAudioFilter):
|
||||||
|
"""Audio filter using the noisereduce library for noise suppression.
|
||||||
|
|
||||||
|
Applies spectral gating noise reduction algorithms to suppress background
|
||||||
|
noise in audio streams. Uses the noisereduce library's default noise
|
||||||
|
reduction parameters.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
|
"""Initialize the noisereduce filter."""
|
||||||
self._filtering = True
|
self._filtering = True
|
||||||
self._sample_rate = 0
|
self._sample_rate = 0
|
||||||
|
|
||||||
async def start(self, sample_rate: int):
|
async def start(self, sample_rate: int):
|
||||||
|
"""Initialize the filter with the transport's sample rate.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: The sample rate of the input transport in Hz.
|
||||||
|
"""
|
||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
|
"""Clean up the filter when stopping."""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def process_frame(self, frame: FilterControlFrame):
|
async def process_frame(self, frame: FilterControlFrame):
|
||||||
|
"""Process control frames to enable/disable filtering.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The control frame containing filter commands.
|
||||||
|
"""
|
||||||
if isinstance(frame, FilterEnableFrame):
|
if isinstance(frame, FilterEnableFrame):
|
||||||
self._filtering = frame.enable
|
self._filtering = frame.enable
|
||||||
|
|
||||||
async def filter(self, audio: bytes) -> bytes:
|
async def filter(self, audio: bytes) -> bytes:
|
||||||
|
"""Apply noise reduction to audio data using spectral gating.
|
||||||
|
|
||||||
|
Converts audio to float32, applies noisereduce processing, and returns
|
||||||
|
the filtered audio clipped to int16 range.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio data as bytes to be filtered.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Noise-reduced audio data as bytes.
|
||||||
|
"""
|
||||||
if not self._filtering:
|
if not self._filtering:
|
||||||
return audio
|
return audio
|
||||||
|
|
||||||
|
|||||||
@@ -4,31 +4,51 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base interruption strategy for determining when users can interrupt bot speech."""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
|
||||||
class BaseInterruptionStrategy(ABC):
|
class BaseInterruptionStrategy(ABC):
|
||||||
"""This is a base class for interruption strategies. Interruption strategies
|
"""Base class for interruption strategies.
|
||||||
|
|
||||||
|
This is a base class for interruption strategies. Interruption strategies
|
||||||
decide when the user can interrupt the bot while the bot is speaking. For
|
decide when the user can interrupt the bot while the bot is speaking. For
|
||||||
example, there could be strategies based on audio volume or strategies based
|
example, there could be strategies based on audio volume or strategies based
|
||||||
on the number of words the user spoke.
|
on the number of words the user spoke.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def append_audio(self, audio: bytes, sample_rate: int):
|
async def append_audio(self, audio: bytes, sample_rate: int):
|
||||||
"""Appends audio to the strategy. Not all strategies handle audio."""
|
"""Append audio data to the strategy for analysis.
|
||||||
|
|
||||||
|
Not all strategies handle audio. Default implementation does nothing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio bytes to append.
|
||||||
|
sample_rate: Sample rate of the audio data in Hz.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def append_text(self, text: str):
|
async def append_text(self, text: str):
|
||||||
"""Appends text to the strategy. Not all strategies handle text."""
|
"""Append text data to the strategy for analysis.
|
||||||
|
|
||||||
|
Not all strategies handle text. Default implementation does nothing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text string to append for analysis.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def should_interrupt(self) -> bool:
|
async def should_interrupt(self) -> bool:
|
||||||
"""This is called when the user stops speaking and it's time to decide
|
"""Determine if the user should interrupt the bot.
|
||||||
|
|
||||||
|
This is called when the user stops speaking and it's time to decide
|
||||||
whether the user should interrupt the bot. The decision will be based on
|
whether the user should interrupt the bot. The decision will be based on
|
||||||
the aggregated audio and/or text.
|
the aggregated audio and/or text.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the user should interrupt the bot, False otherwise.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|||||||
@@ -4,31 +4,47 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Minimum words interruption strategy for word count-based interruptions."""
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
|
from pipecat.audio.interruptions.base_interruption_strategy import BaseInterruptionStrategy
|
||||||
|
|
||||||
|
|
||||||
class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
|
class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
|
||||||
"""This is an interruption strategy based on a minimum number of words said
|
"""Interruption strategy based on minimum number of words spoken.
|
||||||
|
|
||||||
|
This is an interruption strategy based on a minimum number of words said
|
||||||
by the user. That is, the strategy will be true if the user has said at
|
by the user. That is, the strategy will be true if the user has said at
|
||||||
least that amount of words.
|
least that amount of words.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, *, min_words: int):
|
def __init__(self, *, min_words: int):
|
||||||
|
"""Initialize the minimum words interruption strategy.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
min_words: Minimum number of words required to trigger an interruption.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._min_words = min_words
|
self._min_words = min_words
|
||||||
self._text = ""
|
self._text = ""
|
||||||
|
|
||||||
async def append_text(self, text: str):
|
async def append_text(self, text: str):
|
||||||
"""Appends text for later analysis. Not all strategies need to handle
|
"""Append text for word count analysis.
|
||||||
text.
|
|
||||||
|
|
||||||
|
Args:
|
||||||
|
text: Text string to append to the accumulated text.
|
||||||
|
|
||||||
|
Note: Not all strategies need to handle text.
|
||||||
"""
|
"""
|
||||||
self._text += text
|
self._text += text
|
||||||
|
|
||||||
async def should_interrupt(self) -> bool:
|
async def should_interrupt(self) -> bool:
|
||||||
|
"""Check if the minimum word count has been reached.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the user has spoken at least the minimum number of words.
|
||||||
|
"""
|
||||||
word_count = len(self._text.split())
|
word_count = len(self._text.split())
|
||||||
interrupt = word_count >= self._min_words
|
interrupt = word_count >= self._min_words
|
||||||
logger.debug(
|
logger.debug(
|
||||||
@@ -37,4 +53,5 @@ class MinWordsInterruptionStrategy(BaseInterruptionStrategy):
|
|||||||
return interrupt
|
return interrupt
|
||||||
|
|
||||||
async def reset(self):
|
async def reset(self):
|
||||||
|
"""Reset the accumulated text for the next analysis cycle."""
|
||||||
self._text = ""
|
self._text = ""
|
||||||
|
|||||||
@@ -4,50 +4,73 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base audio mixer for output transport integration.
|
||||||
|
|
||||||
|
Provides the abstract base class for audio mixers that can be integrated with
|
||||||
|
output transports to mix incoming audio with generated audio from the mixer.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
from pipecat.frames.frames import MixerControlFrame
|
from pipecat.frames.frames import MixerControlFrame
|
||||||
|
|
||||||
|
|
||||||
class BaseAudioMixer(ABC):
|
class BaseAudioMixer(ABC):
|
||||||
"""This is a base class for output transport audio mixers. If an audio mixer
|
"""Base class for output transport audio mixers.
|
||||||
|
|
||||||
|
This is a base class for output transport audio mixers. If an audio mixer
|
||||||
is provided to the output transport it will be used to mix the audio frames
|
is provided to the output transport it will be used to mix the audio frames
|
||||||
coming into to the transport with the audio generated from the mixer. There
|
coming into to the transport with the audio generated from the mixer. There
|
||||||
are control frames to update mixer settings or to enable or disable the
|
are control frames to update mixer settings or to enable or disable the
|
||||||
mixer at runtime.
|
mixer at runtime.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def start(self, sample_rate: int):
|
async def start(self, sample_rate: int):
|
||||||
"""This will be called from the output transport when the transport is
|
"""Initialize the mixer when the output transport starts.
|
||||||
|
|
||||||
|
This will be called from the output transport when the transport is
|
||||||
started. It can be used to initialize the mixer. The output transport
|
started. It can be used to initialize the mixer. The output transport
|
||||||
sample rate is provided so the mixer can adjust to that sample rate.
|
sample rate is provided so the mixer can adjust to that sample rate.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: The sample rate of the output transport in Hz.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
"""This will be called from the output transport when the transport is
|
"""Clean up the mixer when the output transport stops.
|
||||||
stopping.
|
|
||||||
|
|
||||||
|
This will be called from the output transport when the transport is
|
||||||
|
stopping.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def process_frame(self, frame: MixerControlFrame):
|
async def process_frame(self, frame: MixerControlFrame):
|
||||||
"""This will be called when the output transport receives a
|
"""Process mixer control frames from the transport.
|
||||||
|
|
||||||
|
This will be called when the output transport receives a
|
||||||
MixerControlFrame.
|
MixerControlFrame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The mixer control frame to process.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def mix(self, audio: bytes) -> bytes:
|
async def mix(self, audio: bytes) -> bytes:
|
||||||
"""This is called with the audio that is about to be sent from the
|
"""Mix transport audio with mixer-generated audio.
|
||||||
|
|
||||||
|
This is called with the audio that is about to be sent from the
|
||||||
output transport and that should be mixed with the mixer audio if the
|
output transport and that should be mixed with the mixer audio if the
|
||||||
mixer is enabled.
|
mixer is enabled.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio bytes from the transport to mix.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Mixed audio bytes combining transport and mixer audio.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Soundfile-based audio mixer for file playback integration.
|
||||||
|
|
||||||
|
Provides an audio mixer that combines incoming audio with audio loaded from
|
||||||
|
files using the soundfile library. Supports multiple audio formats and
|
||||||
|
runtime configuration changes.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from typing import Any, Dict, Mapping
|
from typing import Any, Dict, Mapping
|
||||||
|
|
||||||
@@ -24,7 +31,9 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class SoundfileMixer(BaseAudioMixer):
|
class SoundfileMixer(BaseAudioMixer):
|
||||||
"""This is an audio mixer that mixes incoming audio with audio from a
|
"""Audio mixer that combines incoming audio with file-based audio.
|
||||||
|
|
||||||
|
This is an audio mixer that mixes incoming audio with audio from a
|
||||||
file. It uses the soundfile library to load files so it supports multiple
|
file. It uses the soundfile library to load files so it supports multiple
|
||||||
formats. The audio files need to only have one channel (mono) and it needs
|
formats. The audio files need to only have one channel (mono) and it needs
|
||||||
to match the sample rate of the output transport.
|
to match the sample rate of the output transport.
|
||||||
@@ -33,7 +42,6 @@ class SoundfileMixer(BaseAudioMixer):
|
|||||||
`MixerUpdateSettingsFrame` has the following settings available: `sound`
|
`MixerUpdateSettingsFrame` has the following settings available: `sound`
|
||||||
(str) and `volume` (float) to be able to update to a different sound file or
|
(str) and `volume` (float) to be able to update to a different sound file or
|
||||||
to change the volume at runtime.
|
to change the volume at runtime.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -46,6 +54,16 @@ class SoundfileMixer(BaseAudioMixer):
|
|||||||
loop: bool = True,
|
loop: bool = True,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the soundfile mixer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sound_files: Mapping of sound names to file paths for loading.
|
||||||
|
default_sound: Name of the default sound to play initially.
|
||||||
|
volume: Mixing volume level (0.0 to 1.0). Defaults to 0.4.
|
||||||
|
mixing: Whether mixing is initially enabled. Defaults to True.
|
||||||
|
loop: Whether to loop audio files when they end. Defaults to True.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._sound_files = sound_files
|
self._sound_files = sound_files
|
||||||
self._volume = volume
|
self._volume = volume
|
||||||
@@ -58,14 +76,28 @@ class SoundfileMixer(BaseAudioMixer):
|
|||||||
self._loop = loop
|
self._loop = loop
|
||||||
|
|
||||||
async def start(self, sample_rate: int):
|
async def start(self, sample_rate: int):
|
||||||
|
"""Initialize the mixer and load all sound files.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: The sample rate of the output transport in Hz.
|
||||||
|
"""
|
||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
for sound_name, file_name in self._sound_files.items():
|
for sound_name, file_name in self._sound_files.items():
|
||||||
await asyncio.to_thread(self._load_sound_file, sound_name, file_name)
|
await asyncio.to_thread(self._load_sound_file, sound_name, file_name)
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
|
"""Clean up mixer resources.
|
||||||
|
|
||||||
|
Currently performs no cleanup as sound data is managed by garbage collection.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def process_frame(self, frame: MixerControlFrame):
|
async def process_frame(self, frame: MixerControlFrame):
|
||||||
|
"""Process mixer control frames to update settings or enable/disable mixing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The mixer control frame to process.
|
||||||
|
"""
|
||||||
if isinstance(frame, MixerUpdateSettingsFrame):
|
if isinstance(frame, MixerUpdateSettingsFrame):
|
||||||
await self._update_settings(frame)
|
await self._update_settings(frame)
|
||||||
elif isinstance(frame, MixerEnableFrame):
|
elif isinstance(frame, MixerEnableFrame):
|
||||||
@@ -73,12 +105,22 @@ class SoundfileMixer(BaseAudioMixer):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
async def mix(self, audio: bytes) -> bytes:
|
async def mix(self, audio: bytes) -> bytes:
|
||||||
|
"""Mix transport audio with the current sound file.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Raw audio bytes from the transport to mix.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Mixed audio bytes combining transport and file audio.
|
||||||
|
"""
|
||||||
return self._mix_with_sound(audio)
|
return self._mix_with_sound(audio)
|
||||||
|
|
||||||
async def _enable_mixing(self, enable: bool):
|
async def _enable_mixing(self, enable: bool):
|
||||||
|
"""Enable or disable audio mixing."""
|
||||||
self._mixing = enable
|
self._mixing = enable
|
||||||
|
|
||||||
async def _update_settings(self, frame: MixerUpdateSettingsFrame):
|
async def _update_settings(self, frame: MixerUpdateSettingsFrame):
|
||||||
|
"""Update mixer settings from a control frame."""
|
||||||
for setting, value in frame.settings.items():
|
for setting, value in frame.settings.items():
|
||||||
match setting:
|
match setting:
|
||||||
case "sound":
|
case "sound":
|
||||||
@@ -89,6 +131,11 @@ class SoundfileMixer(BaseAudioMixer):
|
|||||||
await self._update_loop(value)
|
await self._update_loop(value)
|
||||||
|
|
||||||
async def _change_sound(self, sound: str):
|
async def _change_sound(self, sound: str):
|
||||||
|
"""Change the currently playing sound file.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sound: Name of the sound file to switch to.
|
||||||
|
"""
|
||||||
if sound in self._sound_files:
|
if sound in self._sound_files:
|
||||||
self._current_sound = sound
|
self._current_sound = sound
|
||||||
self._sound_pos = 0
|
self._sound_pos = 0
|
||||||
@@ -96,12 +143,15 @@ class SoundfileMixer(BaseAudioMixer):
|
|||||||
logger.error(f"Sound {sound} is not available")
|
logger.error(f"Sound {sound} is not available")
|
||||||
|
|
||||||
async def _update_volume(self, volume: float):
|
async def _update_volume(self, volume: float):
|
||||||
|
"""Update the mixing volume level."""
|
||||||
self._volume = volume
|
self._volume = volume
|
||||||
|
|
||||||
async def _update_loop(self, loop: bool):
|
async def _update_loop(self, loop: bool):
|
||||||
|
"""Update the looping behavior."""
|
||||||
self._loop = loop
|
self._loop = loop
|
||||||
|
|
||||||
def _load_sound_file(self, sound_name: str, file_name: str):
|
def _load_sound_file(self, sound_name: str, file_name: str):
|
||||||
|
"""Load an audio file into memory for mixing."""
|
||||||
try:
|
try:
|
||||||
logger.debug(f"Loading mixer sound from {file_name}")
|
logger.debug(f"Loading mixer sound from {file_name}")
|
||||||
sound, sample_rate = sf.read(file_name, dtype="int16")
|
sound, sample_rate = sf.read(file_name, dtype="int16")
|
||||||
@@ -118,10 +168,7 @@ class SoundfileMixer(BaseAudioMixer):
|
|||||||
logger.error(f"Unable to open file {file_name}: {e}")
|
logger.error(f"Unable to open file {file_name}: {e}")
|
||||||
|
|
||||||
def _mix_with_sound(self, audio: bytes):
|
def _mix_with_sound(self, audio: bytes):
|
||||||
"""Mixes raw audio frames with chunks of the same length from the sound
|
"""Mix raw audio frames with chunks of the same length from the sound file."""
|
||||||
file.
|
|
||||||
|
|
||||||
"""
|
|
||||||
if not self._mixing or not self._current_sound in self._sounds:
|
if not self._mixing or not self._current_sound in self._sounds:
|
||||||
return audio
|
return audio
|
||||||
|
|
||||||
|
|||||||
@@ -4,27 +4,35 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base audio resampler interface for Pipecat.
|
||||||
|
|
||||||
|
This module defines the abstract base class for audio resampling implementations,
|
||||||
|
providing a common interface for converting audio between different sample rates.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
|
||||||
class BaseAudioResampler(ABC):
|
class BaseAudioResampler(ABC):
|
||||||
"""Abstract base class for audio resampling. This class defines an
|
"""Abstract base class for audio resampling implementations.
|
||||||
interface for audio resampling implementations.
|
|
||||||
|
This class defines the interface that all audio resampling implementations
|
||||||
|
must follow, providing a standardized way to convert audio data between
|
||||||
|
different sample rates.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
|
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
|
||||||
"""
|
"""Resamples the given audio data to a different sample rate.
|
||||||
Resamples the given audio data to a different sample rate.
|
|
||||||
|
|
||||||
This is an abstract method that must be implemented in subclasses.
|
This is an abstract method that must be implemented in subclasses.
|
||||||
|
|
||||||
Parameters:
|
Args:
|
||||||
audio (bytes): The audio data to be resampled, represented as a byte string.
|
audio: The audio data to be resampled, as raw bytes.
|
||||||
in_rate (int): The original sample rate of the audio data (in Hz).
|
in_rate: The original sample rate of the audio data in Hz.
|
||||||
out_rate (int): The desired sample rate for the resampled audio data (in Hz).
|
out_rate: The desired sample rate for the output audio in Hz.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
bytes: The resampled audio data as a byte string.
|
The resampled audio data as raw bytes.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Resampy-based audio resampler implementation.
|
||||||
|
|
||||||
|
This module provides an audio resampler that uses the resampy library
|
||||||
|
for high-quality audio sample rate conversion.
|
||||||
|
"""
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import resampy
|
import resampy
|
||||||
|
|
||||||
@@ -11,12 +17,31 @@ from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
|
|||||||
|
|
||||||
|
|
||||||
class ResampyResampler(BaseAudioResampler):
|
class ResampyResampler(BaseAudioResampler):
|
||||||
"""Audio resampler implementation using the resampy library."""
|
"""Audio resampler implementation using the resampy library.
|
||||||
|
|
||||||
|
This resampler uses the resampy library's Kaiser windowing filter
|
||||||
|
for high-quality audio resampling with good performance characteristics.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
|
"""Initialize the resampy resampler.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
**kwargs: Additional keyword arguments (currently unused).
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
|
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
|
||||||
|
"""Resample audio data using resampy library.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Input audio data as raw bytes (16-bit signed integers).
|
||||||
|
in_rate: Original sample rate in Hz.
|
||||||
|
out_rate: Target sample rate in Hz.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Resampled audio data as raw bytes (16-bit signed integers).
|
||||||
|
"""
|
||||||
if in_rate == out_rate:
|
if in_rate == out_rate:
|
||||||
return audio
|
return audio
|
||||||
audio_data = np.frombuffer(audio, dtype=np.int16)
|
audio_data = np.frombuffer(audio, dtype=np.int16)
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""SoX-based audio resampler implementation.
|
||||||
|
|
||||||
|
This module provides an audio resampler that uses the SoX resampler library
|
||||||
|
for very high quality audio sample rate conversion.
|
||||||
|
"""
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import soxr
|
import soxr
|
||||||
|
|
||||||
@@ -11,12 +17,32 @@ from pipecat.audio.resamplers.base_audio_resampler import BaseAudioResampler
|
|||||||
|
|
||||||
|
|
||||||
class SOXRAudioResampler(BaseAudioResampler):
|
class SOXRAudioResampler(BaseAudioResampler):
|
||||||
"""Audio resampler implementation using the SoX resampler library."""
|
"""Audio resampler implementation using the SoX resampler library.
|
||||||
|
|
||||||
|
This resampler uses the SoX resampler library configured for very high
|
||||||
|
quality (VHQ) resampling, providing excellent audio quality at the cost
|
||||||
|
of additional computational overhead.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
|
"""Initialize the SoX audio resampler.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
**kwargs: Additional keyword arguments (currently unused).
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
|
async def resample(self, audio: bytes, in_rate: int, out_rate: int) -> bytes:
|
||||||
|
"""Resample audio data using SoX resampler library.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Input audio data as raw bytes (16-bit signed integers).
|
||||||
|
in_rate: Original sample rate in Hz.
|
||||||
|
out_rate: Target sample rate in Hz.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Resampled audio data as raw bytes (16-bit signed integers).
|
||||||
|
"""
|
||||||
if in_rate == out_rate:
|
if in_rate == out_rate:
|
||||||
return audio
|
return audio
|
||||||
audio_data = np.frombuffer(audio, dtype=np.int16)
|
audio_data = np.frombuffer(audio, dtype=np.int16)
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base turn analyzer for determining end-of-turn in audio conversations.
|
||||||
|
|
||||||
|
This module provides the abstract base class and enumeration for analyzing
|
||||||
|
when a user has finished speaking in a conversation.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
@@ -12,6 +18,13 @@ from pipecat.metrics.metrics import MetricsData
|
|||||||
|
|
||||||
|
|
||||||
class EndOfTurnState(Enum):
|
class EndOfTurnState(Enum):
|
||||||
|
"""State enumeration for end-of-turn analysis results.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
COMPLETE: The user has finished their turn and stopped speaking.
|
||||||
|
INCOMPLETE: The user is still speaking or may continue speaking.
|
||||||
|
"""
|
||||||
|
|
||||||
COMPLETE = 1
|
COMPLETE = 1
|
||||||
INCOMPLETE = 2
|
INCOMPLETE = 2
|
||||||
|
|
||||||
@@ -24,6 +37,12 @@ class BaseTurnAnalyzer(ABC):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, *, sample_rate: Optional[int] = None):
|
def __init__(self, *, sample_rate: Optional[int] = None):
|
||||||
|
"""Initialize the turn analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: Optional initial sample rate for audio processing.
|
||||||
|
If provided, this will be used as the fixed sample rate.
|
||||||
|
"""
|
||||||
self._init_sample_rate = sample_rate
|
self._init_sample_rate = sample_rate
|
||||||
self._sample_rate = 0
|
self._sample_rate = 0
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Smart turn analyzer base class using ML models for end-of-turn detection.
|
||||||
|
|
||||||
|
This module provides the base implementation for smart turn analyzers that use
|
||||||
|
machine learning models to determine when a user has finished speaking, going
|
||||||
|
beyond simple silence-based detection.
|
||||||
|
"""
|
||||||
|
|
||||||
import time
|
import time
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from typing import Any, Dict, Optional, Tuple
|
from typing import Any, Dict, Optional, Tuple
|
||||||
@@ -23,6 +30,14 @@ USE_ONLY_LAST_VAD_SEGMENT = True
|
|||||||
|
|
||||||
|
|
||||||
class SmartTurnParams(BaseModel):
|
class SmartTurnParams(BaseModel):
|
||||||
|
"""Configuration parameters for smart turn analysis.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
stop_secs: Maximum silence duration in seconds before ending turn.
|
||||||
|
pre_speech_ms: Milliseconds of audio to include before speech starts.
|
||||||
|
max_duration_secs: Maximum duration in seconds for audio segments.
|
||||||
|
"""
|
||||||
|
|
||||||
stop_secs: float = STOP_SECS
|
stop_secs: float = STOP_SECS
|
||||||
pre_speech_ms: float = PRE_SPEECH_MS
|
pre_speech_ms: float = PRE_SPEECH_MS
|
||||||
max_duration_secs: float = MAX_DURATION_SECONDS
|
max_duration_secs: float = MAX_DURATION_SECONDS
|
||||||
@@ -31,13 +46,28 @@ class SmartTurnParams(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class SmartTurnTimeoutException(Exception):
|
class SmartTurnTimeoutException(Exception):
|
||||||
|
"""Exception raised when smart turn analysis times out."""
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class BaseSmartTurn(BaseTurnAnalyzer):
|
class BaseSmartTurn(BaseTurnAnalyzer):
|
||||||
|
"""Base class for smart turn analyzers using ML models.
|
||||||
|
|
||||||
|
Provides common functionality for smart turn detection including audio
|
||||||
|
buffering, speech tracking, and ML model integration. Subclasses must
|
||||||
|
implement the specific model prediction logic.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self, *, sample_rate: Optional[int] = None, params: Optional[SmartTurnParams] = None
|
self, *, sample_rate: Optional[int] = None, params: Optional[SmartTurnParams] = None
|
||||||
):
|
):
|
||||||
|
"""Initialize the smart turn analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: Optional sample rate for audio processing.
|
||||||
|
params: Configuration parameters for turn analysis behavior.
|
||||||
|
"""
|
||||||
super().__init__(sample_rate=sample_rate)
|
super().__init__(sample_rate=sample_rate)
|
||||||
self._params = params or SmartTurnParams()
|
self._params = params or SmartTurnParams()
|
||||||
# Configuration
|
# Configuration
|
||||||
@@ -50,9 +80,23 @@ class BaseSmartTurn(BaseTurnAnalyzer):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def speech_triggered(self) -> bool:
|
def speech_triggered(self) -> bool:
|
||||||
|
"""Check if speech has been detected and triggered analysis.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if speech has been detected and turn analysis is active.
|
||||||
|
"""
|
||||||
return self._speech_triggered
|
return self._speech_triggered
|
||||||
|
|
||||||
def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
|
def append_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
|
||||||
|
"""Append audio data for turn analysis.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
buffer: Raw audio data bytes to append for analysis.
|
||||||
|
is_speech: Whether the audio buffer contains detected speech.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Current end-of-turn state after processing the audio.
|
||||||
|
"""
|
||||||
# Convert raw audio to float32 format and append to the buffer
|
# Convert raw audio to float32 format and append to the buffer
|
||||||
audio_int16 = np.frombuffer(buffer, dtype=np.int16)
|
audio_int16 = np.frombuffer(buffer, dtype=np.int16)
|
||||||
audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
|
audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
|
||||||
@@ -92,6 +136,12 @@ class BaseSmartTurn(BaseTurnAnalyzer):
|
|||||||
return state
|
return state
|
||||||
|
|
||||||
async def analyze_end_of_turn(self) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
|
async def analyze_end_of_turn(self) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
|
||||||
|
"""Analyze the current audio state to determine if turn has ended.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple containing the end-of-turn state and optional metrics data
|
||||||
|
from the ML model analysis.
|
||||||
|
"""
|
||||||
state, result = await self._process_speech_segment(self._audio_buffer)
|
state, result = await self._process_speech_segment(self._audio_buffer)
|
||||||
if state == EndOfTurnState.COMPLETE or USE_ONLY_LAST_VAD_SEGMENT:
|
if state == EndOfTurnState.COMPLETE or USE_ONLY_LAST_VAD_SEGMENT:
|
||||||
self._clear(state)
|
self._clear(state)
|
||||||
@@ -99,9 +149,11 @@ class BaseSmartTurn(BaseTurnAnalyzer):
|
|||||||
return state, result
|
return state, result
|
||||||
|
|
||||||
def clear(self):
|
def clear(self):
|
||||||
|
"""Reset the turn analyzer to its initial state."""
|
||||||
self._clear(EndOfTurnState.COMPLETE)
|
self._clear(EndOfTurnState.COMPLETE)
|
||||||
|
|
||||||
def _clear(self, turn_state: EndOfTurnState):
|
def _clear(self, turn_state: EndOfTurnState):
|
||||||
|
"""Clear internal state based on turn completion status."""
|
||||||
# If the state is still incomplete, keep the _speech_triggered as True
|
# If the state is still incomplete, keep the _speech_triggered as True
|
||||||
self._speech_triggered = turn_state == EndOfTurnState.INCOMPLETE
|
self._speech_triggered = turn_state == EndOfTurnState.INCOMPLETE
|
||||||
self._audio_buffer = []
|
self._audio_buffer = []
|
||||||
@@ -111,6 +163,7 @@ class BaseSmartTurn(BaseTurnAnalyzer):
|
|||||||
async def _process_speech_segment(
|
async def _process_speech_segment(
|
||||||
self, audio_buffer
|
self, audio_buffer
|
||||||
) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
|
) -> Tuple[EndOfTurnState, Optional[MetricsData]]:
|
||||||
|
"""Process accumulated audio segment using ML model."""
|
||||||
state = EndOfTurnState.INCOMPLETE
|
state = EndOfTurnState.INCOMPLETE
|
||||||
|
|
||||||
if not audio_buffer:
|
if not audio_buffer:
|
||||||
@@ -188,14 +241,5 @@ class BaseSmartTurn(BaseTurnAnalyzer):
|
|||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
||||||
"""Abstract method to predict if a turn has ended based on audio.
|
"""Predict end-of-turn using ML model from audio data."""
|
||||||
|
|
||||||
Args:
|
|
||||||
audio_array: Float32 numpy array of audio samples at 16kHz.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dictionary with:
|
|
||||||
- prediction: 1 if turn is complete, else 0
|
|
||||||
- probability: Confidence of the prediction
|
|
||||||
"""
|
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,16 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Fal.ai smart turn analyzer implementation.
|
||||||
|
|
||||||
|
This module provides a smart turn analyzer that uses Fal.ai's hosted smart-turn model
|
||||||
|
for end-of-turn detection in conversations.
|
||||||
|
|
||||||
|
Note: To learn more about the smart-turn model, visit:
|
||||||
|
- https://fal.ai/models/fal-ai/smart-turn/playground
|
||||||
|
- https://github.com/pipecat-ai/smart-turn
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
@@ -12,6 +22,12 @@ from pipecat.audio.turn.smart_turn.http_smart_turn import HttpSmartTurnAnalyzer
|
|||||||
|
|
||||||
|
|
||||||
class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
|
class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
|
||||||
|
"""Smart turn analyzer using Fal.ai's hosted smart-turn model.
|
||||||
|
|
||||||
|
Extends HttpSmartTurnAnalyzer to provide integration with Fal.ai's
|
||||||
|
smart turn detection API endpoint with proper authentication.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -20,6 +36,14 @@ class FalSmartTurnAnalyzer(HttpSmartTurnAnalyzer):
|
|||||||
api_key: Optional[str] = None,
|
api_key: Optional[str] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the Fal.ai smart turn analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
aiohttp_session: HTTP client session for making API requests.
|
||||||
|
url: Fal.ai API endpoint URL for smart turn detection.
|
||||||
|
api_key: API key for authenticating with Fal.ai service.
|
||||||
|
**kwargs: Additional arguments passed to parent HttpSmartTurnAnalyzer.
|
||||||
|
"""
|
||||||
headers = {}
|
headers = {}
|
||||||
if api_key:
|
if api_key:
|
||||||
headers = {"Authorization": f"Key {api_key}"}
|
headers = {"Authorization": f"Key {api_key}"}
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""HTTP-based smart turn analyzer for remote ML inference.
|
||||||
|
|
||||||
|
This module provides a smart turn analyzer that sends audio data to remote
|
||||||
|
HTTP endpoints for ML-based end-of-turn detection.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import io
|
import io
|
||||||
from typing import Any, Dict, Optional
|
from typing import Any, Dict, Optional
|
||||||
@@ -16,6 +22,12 @@ from pipecat.audio.turn.smart_turn.base_smart_turn import BaseSmartTurn, SmartTu
|
|||||||
|
|
||||||
|
|
||||||
class HttpSmartTurnAnalyzer(BaseSmartTurn):
|
class HttpSmartTurnAnalyzer(BaseSmartTurn):
|
||||||
|
"""Smart turn analyzer using HTTP-based ML inference.
|
||||||
|
|
||||||
|
Sends audio data to remote HTTP endpoints for ML-based end-of-turn
|
||||||
|
prediction. Handles serialization, HTTP communication, and error recovery.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -24,12 +36,21 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
|
|||||||
headers: Optional[Dict[str, str]] = None,
|
headers: Optional[Dict[str, str]] = None,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the HTTP smart turn analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: HTTP endpoint URL for the smart turn ML service.
|
||||||
|
aiohttp_session: HTTP client session for making requests.
|
||||||
|
headers: Optional HTTP headers to include in requests.
|
||||||
|
**kwargs: Additional arguments passed to BaseSmartTurn.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._url = url
|
self._url = url
|
||||||
self._headers = headers or {}
|
self._headers = headers or {}
|
||||||
self._aiohttp_session = aiohttp_session
|
self._aiohttp_session = aiohttp_session
|
||||||
|
|
||||||
def _serialize_array(self, audio_array: np.ndarray) -> bytes:
|
def _serialize_array(self, audio_array: np.ndarray) -> bytes:
|
||||||
|
"""Serialize NumPy audio array to bytes for HTTP transmission."""
|
||||||
logger.trace("Serializing NumPy array to bytes...")
|
logger.trace("Serializing NumPy array to bytes...")
|
||||||
buffer = io.BytesIO()
|
buffer = io.BytesIO()
|
||||||
np.save(buffer, audio_array)
|
np.save(buffer, audio_array)
|
||||||
@@ -38,6 +59,7 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
|
|||||||
return serialized_bytes
|
return serialized_bytes
|
||||||
|
|
||||||
async def _send_raw_request(self, data_bytes: bytes) -> Dict[str, Any]:
|
async def _send_raw_request(self, data_bytes: bytes) -> Dict[str, Any]:
|
||||||
|
"""Send raw audio data to the HTTP endpoint for prediction."""
|
||||||
headers = {"Content-Type": "application/octet-stream"}
|
headers = {"Content-Type": "application/octet-stream"}
|
||||||
headers.update(self._headers)
|
headers.update(self._headers)
|
||||||
|
|
||||||
@@ -83,6 +105,7 @@ class HttpSmartTurnAnalyzer(BaseSmartTurn):
|
|||||||
raise Exception("Failed to send raw request to Daily Smart Turn.")
|
raise Exception("Failed to send raw request to Daily Smart Turn.")
|
||||||
|
|
||||||
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
||||||
|
"""Predict end-of-turn using remote HTTP ML service."""
|
||||||
try:
|
try:
|
||||||
serialized_array = self._serialize_array(audio_array)
|
serialized_array = self._serialize_array(audio_array)
|
||||||
return await self._send_raw_request(serialized_array)
|
return await self._send_raw_request(serialized_array)
|
||||||
|
|||||||
@@ -4,6 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Local CoreML smart turn analyzer for on-device ML inference.
|
||||||
|
|
||||||
|
This module provides a smart turn analyzer that uses CoreML models for
|
||||||
|
local end-of-turn detection without requiring network connectivity.
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
|
|
||||||
@@ -25,7 +30,24 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
|
class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
|
||||||
|
"""Local smart turn analyzer using CoreML models.
|
||||||
|
|
||||||
|
Provides end-of-turn detection using locally-stored CoreML models,
|
||||||
|
enabling offline operation without network dependencies. Optimized
|
||||||
|
for Apple Silicon and other CoreML-compatible hardware.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, *, smart_turn_model_path: str, **kwargs):
|
def __init__(self, *, smart_turn_model_path: str, **kwargs):
|
||||||
|
"""Initialize the local CoreML smart turn analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
smart_turn_model_path: Path to directory containing the CoreML model
|
||||||
|
and feature extractor files.
|
||||||
|
**kwargs: Additional arguments passed to BaseSmartTurn.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If smart_turn_model_path is not provided or model loading fails.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
if not smart_turn_model_path:
|
if not smart_turn_model_path:
|
||||||
@@ -41,6 +63,7 @@ class LocalCoreMLSmartTurnAnalyzer(BaseSmartTurn):
|
|||||||
logger.debug("Loaded Local Smart Turn")
|
logger.debug("Loaded Local Smart Turn")
|
||||||
|
|
||||||
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
||||||
|
"""Predict end-of-turn using local CoreML model."""
|
||||||
inputs = self._turn_processor(
|
inputs = self._turn_processor(
|
||||||
audio_array,
|
audio_array,
|
||||||
sampling_rate=16000,
|
sampling_rate=16000,
|
||||||
|
|||||||
@@ -4,6 +4,11 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Local PyTorch smart turn analyzer for on-device ML inference.
|
||||||
|
|
||||||
|
This module provides a smart turn analyzer that uses PyTorch models for
|
||||||
|
local end-of-turn detection without requiring network connectivity.
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import Any, Dict
|
from typing import Any, Dict
|
||||||
|
|
||||||
@@ -24,7 +29,21 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class LocalSmartTurnAnalyzer(BaseSmartTurn):
|
class LocalSmartTurnAnalyzer(BaseSmartTurn):
|
||||||
|
"""Local smart turn analyzer using PyTorch models.
|
||||||
|
|
||||||
|
Provides end-of-turn detection using locally-stored PyTorch models,
|
||||||
|
enabling offline operation without network dependencies. Uses
|
||||||
|
Wav2Vec2-BERT architecture for audio sequence classification.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, *, smart_turn_model_path: str, **kwargs):
|
def __init__(self, *, smart_turn_model_path: str, **kwargs):
|
||||||
|
"""Initialize the local PyTorch smart turn analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
smart_turn_model_path: Path to directory containing the PyTorch model
|
||||||
|
and feature extractor files. If empty, uses default HuggingFace model.
|
||||||
|
**kwargs: Additional arguments passed to BaseSmartTurn.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
if not smart_turn_model_path:
|
if not smart_turn_model_path:
|
||||||
@@ -46,6 +65,7 @@ class LocalSmartTurnAnalyzer(BaseSmartTurn):
|
|||||||
logger.debug("Loaded Local Smart Turn")
|
logger.debug("Loaded Local Smart Turn")
|
||||||
|
|
||||||
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
async def _predict_endpoint(self, audio_array: np.ndarray) -> Dict[str, Any]:
|
||||||
|
"""Predict end-of-turn using local PyTorch model."""
|
||||||
inputs = self._turn_processor(
|
inputs = self._turn_processor(
|
||||||
audio_array,
|
audio_array,
|
||||||
sampling_rate=16000,
|
sampling_rate=16000,
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Audio utility functions for Pipecat.
|
||||||
|
|
||||||
|
This module provides common audio processing utilities including mixing,
|
||||||
|
format conversion, volume calculation, and codec transformations for
|
||||||
|
various audio formats used in Pipecat pipelines.
|
||||||
|
"""
|
||||||
|
|
||||||
import audioop
|
import audioop
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -15,10 +22,31 @@ from pipecat.audio.resamplers.soxr_resampler import SOXRAudioResampler
|
|||||||
|
|
||||||
|
|
||||||
def create_default_resampler(**kwargs) -> BaseAudioResampler:
|
def create_default_resampler(**kwargs) -> BaseAudioResampler:
|
||||||
|
"""Create a default audio resampler instance.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
**kwargs: Additional keyword arguments passed to the resampler constructor.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A configured SOXRAudioResampler instance.
|
||||||
|
"""
|
||||||
return SOXRAudioResampler(**kwargs)
|
return SOXRAudioResampler(**kwargs)
|
||||||
|
|
||||||
|
|
||||||
def mix_audio(audio1: bytes, audio2: bytes) -> bytes:
|
def mix_audio(audio1: bytes, audio2: bytes) -> bytes:
|
||||||
|
"""Mix two audio streams together by adding their samples.
|
||||||
|
|
||||||
|
Both audio streams are assumed to be 16-bit signed integer PCM data.
|
||||||
|
If the streams have different lengths, the shorter one is zero-padded
|
||||||
|
to match the longer stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio1: First audio stream as raw bytes (16-bit signed integers).
|
||||||
|
audio2: Second audio stream as raw bytes (16-bit signed integers).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Mixed audio data as raw bytes with samples clipped to 16-bit range.
|
||||||
|
"""
|
||||||
data1 = np.frombuffer(audio1, dtype=np.int16)
|
data1 = np.frombuffer(audio1, dtype=np.int16)
|
||||||
data2 = np.frombuffer(audio2, dtype=np.int16)
|
data2 = np.frombuffer(audio2, dtype=np.int16)
|
||||||
|
|
||||||
@@ -37,6 +65,19 @@ def mix_audio(audio1: bytes, audio2: bytes) -> bytes:
|
|||||||
|
|
||||||
|
|
||||||
def interleave_stereo_audio(left_audio: bytes, right_audio: bytes) -> bytes:
|
def interleave_stereo_audio(left_audio: bytes, right_audio: bytes) -> bytes:
|
||||||
|
"""Interleave left and right mono audio channels into stereo audio.
|
||||||
|
|
||||||
|
Takes two mono audio streams and combines them into a single stereo
|
||||||
|
stream by interleaving the samples (L, R, L, R, ...). If the channels
|
||||||
|
have different lengths, both are truncated to the shorter length.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
left_audio: Left channel audio as raw bytes (16-bit signed integers).
|
||||||
|
right_audio: Right channel audio as raw bytes (16-bit signed integers).
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Interleaved stereo audio data as raw bytes.
|
||||||
|
"""
|
||||||
left = np.frombuffer(left_audio, dtype=np.int16)
|
left = np.frombuffer(left_audio, dtype=np.int16)
|
||||||
right = np.frombuffer(right_audio, dtype=np.int16)
|
right = np.frombuffer(right_audio, dtype=np.int16)
|
||||||
|
|
||||||
@@ -50,12 +91,34 @@ def interleave_stereo_audio(left_audio: bytes, right_audio: bytes) -> bytes:
|
|||||||
|
|
||||||
|
|
||||||
def normalize_value(value, min_value, max_value):
|
def normalize_value(value, min_value, max_value):
|
||||||
|
"""Normalize a value to the range [0, 1] and clamp it to bounds.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
value: The value to normalize.
|
||||||
|
min_value: The minimum value of the input range.
|
||||||
|
max_value: The maximum value of the input range.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Normalized value clamped to the range [0, 1].
|
||||||
|
"""
|
||||||
normalized = (value - min_value) / (max_value - min_value)
|
normalized = (value - min_value) / (max_value - min_value)
|
||||||
normalized_clamped = max(0, min(1, normalized))
|
normalized_clamped = max(0, min(1, normalized))
|
||||||
return normalized_clamped
|
return normalized_clamped
|
||||||
|
|
||||||
|
|
||||||
def calculate_audio_volume(audio: bytes, sample_rate: int) -> float:
|
def calculate_audio_volume(audio: bytes, sample_rate: int) -> float:
|
||||||
|
"""Calculate the loudness level of audio data using EBU R128 standard.
|
||||||
|
|
||||||
|
Uses the pyloudnorm library to calculate integrated loudness according
|
||||||
|
to the EBU R128 recommendation, then normalizes the result to [0, 1].
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio: Audio data as raw bytes (16-bit signed integers).
|
||||||
|
sample_rate: Sample rate of the audio in Hz.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Normalized loudness value between 0 (quiet) and 1 (loud).
|
||||||
|
"""
|
||||||
audio_np = np.frombuffer(audio, dtype=np.int16)
|
audio_np = np.frombuffer(audio, dtype=np.int16)
|
||||||
audio_float = audio_np.astype(np.float64)
|
audio_float = audio_np.astype(np.float64)
|
||||||
|
|
||||||
@@ -71,12 +134,37 @@ def calculate_audio_volume(audio: bytes, sample_rate: int) -> float:
|
|||||||
|
|
||||||
|
|
||||||
def exp_smoothing(value: float, prev_value: float, factor: float) -> float:
|
def exp_smoothing(value: float, prev_value: float, factor: float) -> float:
|
||||||
|
"""Apply exponential smoothing to a value.
|
||||||
|
|
||||||
|
Exponential smoothing is used to reduce noise in time-series data by
|
||||||
|
giving more weight to recent values while still considering historical data.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
value: The new value to incorporate.
|
||||||
|
prev_value: The previous smoothed value.
|
||||||
|
factor: Smoothing factor between 0 and 1. Higher values give more
|
||||||
|
weight to the new value.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The exponentially smoothed value.
|
||||||
|
"""
|
||||||
return prev_value + factor * (value - prev_value)
|
return prev_value + factor * (value - prev_value)
|
||||||
|
|
||||||
|
|
||||||
async def ulaw_to_pcm(
|
async def ulaw_to_pcm(
|
||||||
ulaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
|
ulaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
|
||||||
):
|
):
|
||||||
|
"""Convert μ-law encoded audio to PCM and optionally resample.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
ulaw_bytes: μ-law encoded audio data as raw bytes.
|
||||||
|
in_rate: Original sample rate of the μ-law audio in Hz.
|
||||||
|
out_rate: Desired output sample rate in Hz.
|
||||||
|
resampler: Audio resampler instance for rate conversion.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
PCM audio data as raw bytes at the specified output rate.
|
||||||
|
"""
|
||||||
# Convert μ-law to PCM
|
# Convert μ-law to PCM
|
||||||
in_pcm_bytes = audioop.ulaw2lin(ulaw_bytes, 2)
|
in_pcm_bytes = audioop.ulaw2lin(ulaw_bytes, 2)
|
||||||
|
|
||||||
@@ -87,6 +175,17 @@ async def ulaw_to_pcm(
|
|||||||
|
|
||||||
|
|
||||||
async def pcm_to_ulaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
|
async def pcm_to_ulaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
|
||||||
|
"""Convert PCM audio to μ-law encoding and optionally resample.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pcm_bytes: PCM audio data as raw bytes (16-bit signed integers).
|
||||||
|
in_rate: Original sample rate of the PCM audio in Hz.
|
||||||
|
out_rate: Desired output sample rate in Hz.
|
||||||
|
resampler: Audio resampler instance for rate conversion.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
μ-law encoded audio data as raw bytes at the specified output rate.
|
||||||
|
"""
|
||||||
# Resample
|
# Resample
|
||||||
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
|
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
|
||||||
|
|
||||||
@@ -99,6 +198,17 @@ async def pcm_to_ulaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler:
|
|||||||
async def alaw_to_pcm(
|
async def alaw_to_pcm(
|
||||||
alaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
|
alaw_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler
|
||||||
) -> bytes:
|
) -> bytes:
|
||||||
|
"""Convert A-law encoded audio to PCM and optionally resample.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
alaw_bytes: A-law encoded audio data as raw bytes.
|
||||||
|
in_rate: Original sample rate of the A-law audio in Hz.
|
||||||
|
out_rate: Desired output sample rate in Hz.
|
||||||
|
resampler: Audio resampler instance for rate conversion.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
PCM audio data as raw bytes at the specified output rate.
|
||||||
|
"""
|
||||||
# Convert a-law to PCM
|
# Convert a-law to PCM
|
||||||
in_pcm_bytes = audioop.alaw2lin(alaw_bytes, 2)
|
in_pcm_bytes = audioop.alaw2lin(alaw_bytes, 2)
|
||||||
|
|
||||||
@@ -109,6 +219,17 @@ async def alaw_to_pcm(
|
|||||||
|
|
||||||
|
|
||||||
async def pcm_to_alaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
|
async def pcm_to_alaw(pcm_bytes: bytes, in_rate: int, out_rate: int, resampler: BaseAudioResampler):
|
||||||
|
"""Convert PCM audio to A-law encoding and optionally resample.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pcm_bytes: PCM audio data as raw bytes (16-bit signed integers).
|
||||||
|
in_rate: Original sample rate of the PCM audio in Hz.
|
||||||
|
out_rate: Desired output sample rate in Hz.
|
||||||
|
resampler: Audio resampler instance for rate conversion.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A-law encoded audio data as raw bytes at the specified output rate.
|
||||||
|
"""
|
||||||
# Resample
|
# Resample
|
||||||
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
|
in_pcm_bytes = await resampler.resample(pcm_bytes, in_rate, out_rate)
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Silero Voice Activity Detection (VAD) implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides a VAD analyzer based on the Silero VAD ONNX model,
|
||||||
|
which can detect voice activity in audio streams with high accuracy.
|
||||||
|
Supports 8kHz and 16kHz sample rates.
|
||||||
|
"""
|
||||||
|
|
||||||
import time
|
import time
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
@@ -25,7 +32,20 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class SileroOnnxModel:
|
class SileroOnnxModel:
|
||||||
|
"""ONNX runtime wrapper for the Silero VAD model.
|
||||||
|
|
||||||
|
Provides voice activity detection using the pre-trained Silero VAD model
|
||||||
|
with ONNX runtime for efficient inference. Handles model state management
|
||||||
|
and input validation for audio processing.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, path, force_onnx_cpu=True):
|
def __init__(self, path, force_onnx_cpu=True):
|
||||||
|
"""Initialize the Silero ONNX model.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
path: Path to the ONNX model file.
|
||||||
|
force_onnx_cpu: Whether to force CPU execution provider.
|
||||||
|
"""
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
global np
|
global np
|
||||||
@@ -45,6 +65,7 @@ class SileroOnnxModel:
|
|||||||
self.sample_rates = [8000, 16000]
|
self.sample_rates = [8000, 16000]
|
||||||
|
|
||||||
def _validate_input(self, x, sr: int):
|
def _validate_input(self, x, sr: int):
|
||||||
|
"""Validate and preprocess input audio data."""
|
||||||
if np.ndim(x) == 1:
|
if np.ndim(x) == 1:
|
||||||
x = np.expand_dims(x, 0)
|
x = np.expand_dims(x, 0)
|
||||||
if np.ndim(x) > 2:
|
if np.ndim(x) > 2:
|
||||||
@@ -60,12 +81,18 @@ class SileroOnnxModel:
|
|||||||
return x, sr
|
return x, sr
|
||||||
|
|
||||||
def reset_states(self, batch_size=1):
|
def reset_states(self, batch_size=1):
|
||||||
|
"""Reset the internal model states.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
batch_size: Batch size for state initialization. Defaults to 1.
|
||||||
|
"""
|
||||||
self._state = np.zeros((2, batch_size, 128), dtype="float32")
|
self._state = np.zeros((2, batch_size, 128), dtype="float32")
|
||||||
self._context = np.zeros((batch_size, 0), dtype="float32")
|
self._context = np.zeros((batch_size, 0), dtype="float32")
|
||||||
self._last_sr = 0
|
self._last_sr = 0
|
||||||
self._last_batch_size = 0
|
self._last_batch_size = 0
|
||||||
|
|
||||||
def __call__(self, x, sr: int):
|
def __call__(self, x, sr: int):
|
||||||
|
"""Process audio input through the VAD model."""
|
||||||
x, sr = self._validate_input(x, sr)
|
x, sr = self._validate_input(x, sr)
|
||||||
num_samples = 512 if sr == 16000 else 256
|
num_samples = 512 if sr == 16000 else 256
|
||||||
|
|
||||||
@@ -105,7 +132,20 @@ class SileroOnnxModel:
|
|||||||
|
|
||||||
|
|
||||||
class SileroVADAnalyzer(VADAnalyzer):
|
class SileroVADAnalyzer(VADAnalyzer):
|
||||||
|
"""Voice Activity Detection analyzer using the Silero VAD model.
|
||||||
|
|
||||||
|
Implements VAD analysis using the pre-trained Silero ONNX model for
|
||||||
|
accurate voice activity detection. Supports 8kHz and 16kHz sample rates
|
||||||
|
with automatic model state management and periodic resets.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
|
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
|
||||||
|
"""Initialize the Silero VAD analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: Audio sample rate (8000 or 16000 Hz). If None, will be set later.
|
||||||
|
params: VAD parameters for detection thresholds and timing.
|
||||||
|
"""
|
||||||
super().__init__(sample_rate=sample_rate, params=params)
|
super().__init__(sample_rate=sample_rate, params=params)
|
||||||
|
|
||||||
logger.debug("Loading Silero VAD model...")
|
logger.debug("Loading Silero VAD model...")
|
||||||
@@ -137,6 +177,14 @@ class SileroVADAnalyzer(VADAnalyzer):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def set_sample_rate(self, sample_rate: int):
|
def set_sample_rate(self, sample_rate: int):
|
||||||
|
"""Set the sample rate for audio processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: Audio sample rate (must be 8000 or 16000 Hz).
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If sample rate is not 8000 or 16000 Hz.
|
||||||
|
"""
|
||||||
if sample_rate != 16000 and sample_rate != 8000:
|
if sample_rate != 16000 and sample_rate != 8000:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Silero VAD sample rate needs to be 16000 or 8000 (sample rate: {sample_rate})"
|
f"Silero VAD sample rate needs to be 16000 or 8000 (sample rate: {sample_rate})"
|
||||||
@@ -145,9 +193,22 @@ class SileroVADAnalyzer(VADAnalyzer):
|
|||||||
super().set_sample_rate(sample_rate)
|
super().set_sample_rate(sample_rate)
|
||||||
|
|
||||||
def num_frames_required(self) -> int:
|
def num_frames_required(self) -> int:
|
||||||
|
"""Get the number of audio frames required for VAD analysis.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Number of frames required (512 for 16kHz, 256 for 8kHz).
|
||||||
|
"""
|
||||||
return 512 if self.sample_rate == 16000 else 256
|
return 512 if self.sample_rate == 16000 else 256
|
||||||
|
|
||||||
def voice_confidence(self, buffer) -> float:
|
def voice_confidence(self, buffer) -> float:
|
||||||
|
"""Calculate voice activity confidence for the given audio buffer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
buffer: Audio buffer to analyze.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Voice confidence score between 0.0 and 1.0.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
audio_int16 = np.frombuffer(buffer, np.int16)
|
audio_int16 = np.frombuffer(buffer, np.int16)
|
||||||
# Divide by 32768 because we have signed 16-bit data.
|
# Divide by 32768 because we have signed 16-bit data.
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Voice Activity Detection (VAD) analyzer base classes and utilities.
|
||||||
|
|
||||||
|
This module provides the abstract base class for VAD analyzers and associated
|
||||||
|
data structures for voice activity detection in audio streams. Includes state
|
||||||
|
management, parameter configuration, and audio analysis framework.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -20,6 +27,15 @@ VAD_MIN_VOLUME = 0.6
|
|||||||
|
|
||||||
|
|
||||||
class VADState(Enum):
|
class VADState(Enum):
|
||||||
|
"""Voice Activity Detection states.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
QUIET: No voice activity detected.
|
||||||
|
STARTING: Voice activity beginning, transitioning from quiet.
|
||||||
|
SPEAKING: Active voice detected and confirmed.
|
||||||
|
STOPPING: Voice activity ending, transitioning to quiet.
|
||||||
|
"""
|
||||||
|
|
||||||
QUIET = 1
|
QUIET = 1
|
||||||
STARTING = 2
|
STARTING = 2
|
||||||
SPEAKING = 3
|
SPEAKING = 3
|
||||||
@@ -27,6 +43,15 @@ class VADState(Enum):
|
|||||||
|
|
||||||
|
|
||||||
class VADParams(BaseModel):
|
class VADParams(BaseModel):
|
||||||
|
"""Configuration parameters for Voice Activity Detection.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
confidence: Minimum confidence threshold for voice detection.
|
||||||
|
start_secs: Duration to wait before confirming voice start.
|
||||||
|
stop_secs: Duration to wait before confirming voice stop.
|
||||||
|
min_volume: Minimum audio volume threshold for voice detection.
|
||||||
|
"""
|
||||||
|
|
||||||
confidence: float = VAD_CONFIDENCE
|
confidence: float = VAD_CONFIDENCE
|
||||||
start_secs: float = VAD_START_SECS
|
start_secs: float = VAD_START_SECS
|
||||||
stop_secs: float = VAD_STOP_SECS
|
stop_secs: float = VAD_STOP_SECS
|
||||||
@@ -34,7 +59,20 @@ class VADParams(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class VADAnalyzer(ABC):
|
class VADAnalyzer(ABC):
|
||||||
|
"""Abstract base class for Voice Activity Detection analyzers.
|
||||||
|
|
||||||
|
Provides the framework for implementing VAD analysis with configurable
|
||||||
|
parameters, state management, and audio processing capabilities.
|
||||||
|
Subclasses must implement the core voice confidence calculation.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
|
def __init__(self, *, sample_rate: Optional[int] = None, params: Optional[VADParams] = None):
|
||||||
|
"""Initialize the VAD analyzer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: Audio sample rate in Hz. If None, will be set later.
|
||||||
|
params: VAD parameters for detection configuration.
|
||||||
|
"""
|
||||||
self._init_sample_rate = sample_rate
|
self._init_sample_rate = sample_rate
|
||||||
self._sample_rate = 0
|
self._sample_rate = 0
|
||||||
self._params = params or VADParams()
|
self._params = params or VADParams()
|
||||||
@@ -48,29 +86,67 @@ class VADAnalyzer(ABC):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def sample_rate(self) -> int:
|
def sample_rate(self) -> int:
|
||||||
|
"""Get the current sample rate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Current audio sample rate in Hz.
|
||||||
|
"""
|
||||||
return self._sample_rate
|
return self._sample_rate
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def num_channels(self) -> int:
|
def num_channels(self) -> int:
|
||||||
|
"""Get the number of audio channels.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Number of audio channels (always 1 for mono).
|
||||||
|
"""
|
||||||
return self._num_channels
|
return self._num_channels
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def params(self) -> VADParams:
|
def params(self) -> VADParams:
|
||||||
|
"""Get the current VAD parameters.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Current VAD configuration parameters.
|
||||||
|
"""
|
||||||
return self._params
|
return self._params
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def num_frames_required(self) -> int:
|
def num_frames_required(self) -> int:
|
||||||
|
"""Get the number of audio frames required for analysis.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Number of frames needed for VAD processing.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def voice_confidence(self, buffer) -> float:
|
def voice_confidence(self, buffer) -> float:
|
||||||
|
"""Calculate voice activity confidence for the given audio buffer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
buffer: Audio buffer to analyze.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Voice confidence score between 0.0 and 1.0.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def set_sample_rate(self, sample_rate: int):
|
def set_sample_rate(self, sample_rate: int):
|
||||||
|
"""Set the sample rate for audio processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: Audio sample rate in Hz.
|
||||||
|
"""
|
||||||
self._sample_rate = self._init_sample_rate or sample_rate
|
self._sample_rate = self._init_sample_rate or sample_rate
|
||||||
self.set_params(self._params)
|
self.set_params(self._params)
|
||||||
|
|
||||||
def set_params(self, params: VADParams):
|
def set_params(self, params: VADParams):
|
||||||
|
"""Set VAD parameters and recalculate internal values.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: VAD parameters for detection configuration.
|
||||||
|
"""
|
||||||
logger.debug(f"Setting VAD params to: {params}")
|
logger.debug(f"Setting VAD params to: {params}")
|
||||||
self._params = params
|
self._params = params
|
||||||
self._vad_frames = self.num_frames_required()
|
self._vad_frames = self.num_frames_required()
|
||||||
@@ -85,10 +161,22 @@ class VADAnalyzer(ABC):
|
|||||||
self._vad_state: VADState = VADState.QUIET
|
self._vad_state: VADState = VADState.QUIET
|
||||||
|
|
||||||
def _get_smoothed_volume(self, audio: bytes) -> float:
|
def _get_smoothed_volume(self, audio: bytes) -> float:
|
||||||
|
"""Calculate smoothed audio volume using exponential smoothing."""
|
||||||
volume = calculate_audio_volume(audio, self.sample_rate)
|
volume = calculate_audio_volume(audio, self.sample_rate)
|
||||||
return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
|
return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
|
||||||
|
|
||||||
def analyze_audio(self, buffer) -> VADState:
|
def analyze_audio(self, buffer) -> VADState:
|
||||||
|
"""Analyze audio buffer and return current VAD state.
|
||||||
|
|
||||||
|
Processes incoming audio data, maintains internal state, and determines
|
||||||
|
voice activity status based on confidence and volume thresholds.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
buffer: Audio buffer to analyze.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Current VAD state after processing the buffer.
|
||||||
|
"""
|
||||||
self._vad_buffer += buffer
|
self._vad_buffer += buffer
|
||||||
|
|
||||||
num_required_bytes = self._vad_frames_num_bytes
|
num_required_bytes = self._vad_frames_num_bytes
|
||||||
|
|||||||
@@ -4,14 +4,33 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base clock interface for Pipecat timing operations."""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
|
||||||
class BaseClock(ABC):
|
class BaseClock(ABC):
|
||||||
|
"""Abstract base class for clock implementations.
|
||||||
|
|
||||||
|
Provides a common interface for timing operations used in Pipecat
|
||||||
|
for synchronization, scheduling, and time-based processing.
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_time(self) -> int:
|
def get_time(self) -> int:
|
||||||
|
"""Get the current time value.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The current time as an integer value. The specific unit and
|
||||||
|
reference point depend on the concrete implementation.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def start(self):
|
def start(self):
|
||||||
|
"""Start or initialize the clock.
|
||||||
|
|
||||||
|
Performs any necessary initialization or starts the timing mechanism.
|
||||||
|
This method should be called before using get_time().
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,17 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""System clock implementation for Pipecat."""
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from pipecat.clocks.base_clock import BaseClock
|
from pipecat.clocks.base_clock import BaseClock
|
||||||
|
|
||||||
|
|
||||||
class SystemClock(BaseClock):
|
class SystemClock(BaseClock):
|
||||||
|
"""A monotonic clock implementation using system time.
|
||||||
|
|
||||||
|
Provides high-precision timing using the system's monotonic clock,
|
||||||
|
which is not affected by system clock adjustments and is suitable
|
||||||
|
for measuring elapsed time in real-time applications.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
"""Initialize the system clock.
|
||||||
|
|
||||||
|
The clock starts in an uninitialized state and must be started
|
||||||
|
explicitly using the start() method before time measurement begins.
|
||||||
|
"""
|
||||||
self._time = 0
|
self._time = 0
|
||||||
|
|
||||||
def get_time(self) -> int:
|
def get_time(self) -> int:
|
||||||
|
"""Get the elapsed time since the clock was started.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The elapsed time in nanoseconds since start() was called.
|
||||||
|
Returns 0 if the clock has not been started yet.
|
||||||
|
"""
|
||||||
return time.monotonic_ns() - self._time if self._time > 0 else 0
|
return time.monotonic_ns() - self._time if self._time > 0 else 0
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
|
"""Start the clock and begin time measurement.
|
||||||
|
|
||||||
|
Records the current monotonic time as the reference point
|
||||||
|
for all subsequent get_time() calls.
|
||||||
|
"""
|
||||||
self._time = time.monotonic_ns()
|
self._time = time.monotonic_ns()
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Daily.co room configuration utilities for Pipecat examples."""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -14,6 +16,17 @@ from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
|
|||||||
|
|
||||||
|
|
||||||
async def configure(aiohttp_session: aiohttp.ClientSession):
|
async def configure(aiohttp_session: aiohttp.ClientSession):
|
||||||
|
"""Configure Daily.co room URL and token from arguments or environment.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
aiohttp_session: HTTP session for making API requests.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple containing the room URL and authentication token.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If room URL or API key are not provided.
|
||||||
|
"""
|
||||||
(url, token, _) = await configure_with_args(aiohttp_session)
|
(url, token, _) = await configure_with_args(aiohttp_session)
|
||||||
return (url, token)
|
return (url, token)
|
||||||
|
|
||||||
@@ -21,6 +34,18 @@ async def configure(aiohttp_session: aiohttp.ClientSession):
|
|||||||
async def configure_with_args(
|
async def configure_with_args(
|
||||||
aiohttp_session: aiohttp.ClientSession, parser: Optional[argparse.ArgumentParser] = None
|
aiohttp_session: aiohttp.ClientSession, parser: Optional[argparse.ArgumentParser] = None
|
||||||
):
|
):
|
||||||
|
"""Configure Daily.co room with command-line argument parsing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
aiohttp_session: HTTP session for making API requests.
|
||||||
|
parser: Optional argument parser. If None, creates a default one.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple containing room URL, authentication token, and parsed arguments.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If room URL or API key are not provided via arguments or environment.
|
||||||
|
"""
|
||||||
if not parser:
|
if not parser:
|
||||||
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
|
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Pipecat example runner with support for multiple transport types.
|
||||||
|
|
||||||
|
This module provides a unified interface for running Pipecat examples across
|
||||||
|
different transport types including Daily.co, WebRTC, and Twilio. It handles
|
||||||
|
setup, configuration, and lifecycle management for each transport type.
|
||||||
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
@@ -35,6 +42,15 @@ load_dotenv(override=True)
|
|||||||
|
|
||||||
|
|
||||||
def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
|
def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
|
||||||
|
"""Get client identifier from transport-specific client object.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The transport instance.
|
||||||
|
client: Transport-specific client object.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Client identifier string, empty if transport not supported.
|
||||||
|
"""
|
||||||
if isinstance(transport, SmallWebRTCTransport):
|
if isinstance(transport, SmallWebRTCTransport):
|
||||||
return client.pc_id
|
return client.pc_id
|
||||||
elif isinstance(transport, DailyTransport):
|
elif isinstance(transport, DailyTransport):
|
||||||
@@ -46,6 +62,13 @@ def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
|
|||||||
async def maybe_capture_participant_camera(
|
async def maybe_capture_participant_camera(
|
||||||
transport: BaseTransport, client: Any, framerate: int = 0
|
transport: BaseTransport, client: Any, framerate: int = 0
|
||||||
):
|
):
|
||||||
|
"""Capture participant camera video if transport supports it.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The transport instance.
|
||||||
|
client: Transport-specific client object.
|
||||||
|
framerate: Video capture framerate. Defaults to 0 (auto).
|
||||||
|
"""
|
||||||
if isinstance(transport, DailyTransport):
|
if isinstance(transport, DailyTransport):
|
||||||
await transport.capture_participant_video(
|
await transport.capture_participant_video(
|
||||||
client["id"], framerate=framerate, video_source="camera"
|
client["id"], framerate=framerate, video_source="camera"
|
||||||
@@ -55,6 +78,13 @@ async def maybe_capture_participant_camera(
|
|||||||
async def maybe_capture_participant_screen(
|
async def maybe_capture_participant_screen(
|
||||||
transport: BaseTransport, client: Any, framerate: int = 0
|
transport: BaseTransport, client: Any, framerate: int = 0
|
||||||
):
|
):
|
||||||
|
"""Capture participant screen video if transport supports it.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The transport instance.
|
||||||
|
client: Transport-specific client object.
|
||||||
|
framerate: Video capture framerate. Defaults to 0 (auto).
|
||||||
|
"""
|
||||||
if isinstance(transport, DailyTransport):
|
if isinstance(transport, DailyTransport):
|
||||||
await transport.capture_participant_video(
|
await transport.capture_participant_video(
|
||||||
client["id"], framerate=framerate, video_source="screenVideo"
|
client["id"], framerate=framerate, video_source="screenVideo"
|
||||||
@@ -66,6 +96,13 @@ def run_example_daily(
|
|||||||
args: argparse.Namespace,
|
args: argparse.Namespace,
|
||||||
transport_params: Mapping[str, Callable] = {},
|
transport_params: Mapping[str, Callable] = {},
|
||||||
):
|
):
|
||||||
|
"""Run example using Daily.co transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
run_example: The example function to run.
|
||||||
|
args: Parsed command-line arguments.
|
||||||
|
transport_params: Mapping of transport names to parameter factory functions.
|
||||||
|
"""
|
||||||
logger.info("Running example with DailyTransport...")
|
logger.info("Running example with DailyTransport...")
|
||||||
|
|
||||||
from pipecat.examples.daily_runner import configure
|
from pipecat.examples.daily_runner import configure
|
||||||
@@ -87,6 +124,13 @@ def run_example_webrtc(
|
|||||||
args: argparse.Namespace,
|
args: argparse.Namespace,
|
||||||
transport_params: Mapping[str, Callable] = {},
|
transport_params: Mapping[str, Callable] = {},
|
||||||
):
|
):
|
||||||
|
"""Run example using WebRTC transport with FastAPI server.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
run_example: The example function to run.
|
||||||
|
args: Parsed command-line arguments.
|
||||||
|
transport_params: Mapping of transport names to parameter factory functions.
|
||||||
|
"""
|
||||||
logger.info("Running example with SmallWebRTCTransport...")
|
logger.info("Running example with SmallWebRTCTransport...")
|
||||||
|
|
||||||
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
|
from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
|
||||||
@@ -107,10 +151,20 @@ def run_example_webrtc(
|
|||||||
|
|
||||||
@app.get("/", include_in_schema=False)
|
@app.get("/", include_in_schema=False)
|
||||||
async def root_redirect():
|
async def root_redirect():
|
||||||
|
"""Redirect root requests to client interface."""
|
||||||
return RedirectResponse(url="/client/")
|
return RedirectResponse(url="/client/")
|
||||||
|
|
||||||
@app.post("/api/offer")
|
@app.post("/api/offer")
|
||||||
async def offer(request: dict, background_tasks: BackgroundTasks):
|
async def offer(request: dict, background_tasks: BackgroundTasks):
|
||||||
|
"""Handle WebRTC offer requests and manage peer connections.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
request: WebRTC offer request containing SDP and connection details.
|
||||||
|
background_tasks: FastAPI background tasks for running examples.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
WebRTC answer with connection details.
|
||||||
|
"""
|
||||||
pc_id = request.get("pc_id")
|
pc_id = request.get("pc_id")
|
||||||
|
|
||||||
if pc_id and pc_id in pcs_map:
|
if pc_id and pc_id in pcs_map:
|
||||||
@@ -127,6 +181,11 @@ def run_example_webrtc(
|
|||||||
|
|
||||||
@pipecat_connection.event_handler("closed")
|
@pipecat_connection.event_handler("closed")
|
||||||
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
|
async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
|
||||||
|
"""Handle WebRTC connection closure and cleanup.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
webrtc_connection: The closed WebRTC connection.
|
||||||
|
"""
|
||||||
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
|
logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
|
||||||
pcs_map.pop(webrtc_connection.pc_id, None)
|
pcs_map.pop(webrtc_connection.pc_id, None)
|
||||||
|
|
||||||
@@ -143,6 +202,11 @@ def run_example_webrtc(
|
|||||||
|
|
||||||
@asynccontextmanager
|
@asynccontextmanager
|
||||||
async def lifespan(app: FastAPI):
|
async def lifespan(app: FastAPI):
|
||||||
|
"""Manage FastAPI application lifecycle and cleanup connections.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
app: The FastAPI application instance.
|
||||||
|
"""
|
||||||
yield # Run app
|
yield # Run app
|
||||||
coros = [pc.disconnect() for pc in pcs_map.values()]
|
coros = [pc.disconnect() for pc in pcs_map.values()]
|
||||||
await asyncio.gather(*coros)
|
await asyncio.gather(*coros)
|
||||||
@@ -156,6 +220,13 @@ def run_example_twilio(
|
|||||||
args: argparse.Namespace,
|
args: argparse.Namespace,
|
||||||
transport_params: Mapping[str, Callable] = {},
|
transport_params: Mapping[str, Callable] = {},
|
||||||
):
|
):
|
||||||
|
"""Run example using Twilio transport with FastAPI WebSocket server.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
run_example: The example function to run.
|
||||||
|
args: Parsed command-line arguments.
|
||||||
|
transport_params: Mapping of transport names to parameter factory functions.
|
||||||
|
"""
|
||||||
logger.info("Running example with FastAPIWebsocketTransport (Twilio)...")
|
logger.info("Running example with FastAPIWebsocketTransport (Twilio)...")
|
||||||
|
|
||||||
app = FastAPI()
|
app = FastAPI()
|
||||||
@@ -170,6 +241,11 @@ def run_example_twilio(
|
|||||||
|
|
||||||
@app.post("/")
|
@app.post("/")
|
||||||
async def start_call():
|
async def start_call():
|
||||||
|
"""Handle Twilio webhook and return TwiML response.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
TwiML XML response directing call to WebSocket stream.
|
||||||
|
"""
|
||||||
logger.debug("POST TwiML")
|
logger.debug("POST TwiML")
|
||||||
|
|
||||||
xml_content = f"""<?xml version="1.0" encoding="UTF-8"?>
|
xml_content = f"""<?xml version="1.0" encoding="UTF-8"?>
|
||||||
@@ -184,6 +260,11 @@ def run_example_twilio(
|
|||||||
|
|
||||||
@app.websocket("/ws")
|
@app.websocket("/ws")
|
||||||
async def websocket_endpoint(websocket: WebSocket):
|
async def websocket_endpoint(websocket: WebSocket):
|
||||||
|
"""Handle Twilio WebSocket connections for voice streaming.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
websocket: The WebSocket connection from Twilio.
|
||||||
|
"""
|
||||||
await websocket.accept()
|
await websocket.accept()
|
||||||
|
|
||||||
logger.debug("WebSocket connection accepted")
|
logger.debug("WebSocket connection accepted")
|
||||||
@@ -216,6 +297,13 @@ def run_main(
|
|||||||
args: argparse.Namespace,
|
args: argparse.Namespace,
|
||||||
transport_params: Mapping[str, Callable] = {},
|
transport_params: Mapping[str, Callable] = {},
|
||||||
):
|
):
|
||||||
|
"""Run the example with the specified transport type.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
run_example: The example function to run.
|
||||||
|
args: Parsed command-line arguments.
|
||||||
|
transport_params: Mapping of transport names to parameter factory functions.
|
||||||
|
"""
|
||||||
if args.transport not in transport_params:
|
if args.transport not in transport_params:
|
||||||
logger.error(f"Transport '{args.transport}' not supported by this example")
|
logger.error(f"Transport '{args.transport}' not supported by this example")
|
||||||
return
|
return
|
||||||
@@ -235,6 +323,13 @@ def main(
|
|||||||
parser: Optional[argparse.ArgumentParser] = None,
|
parser: Optional[argparse.ArgumentParser] = None,
|
||||||
transport_params: Mapping[str, Callable] = {},
|
transport_params: Mapping[str, Callable] = {},
|
||||||
):
|
):
|
||||||
|
"""Main entry point for running Pipecat examples with transport selection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
run_example: The example function to run.
|
||||||
|
parser: Optional argument parser. If None, creates a default one.
|
||||||
|
transport_params: Mapping of transport names to parameter factory functions.
|
||||||
|
"""
|
||||||
if not parser:
|
if not parser:
|
||||||
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
|
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,22 +1,64 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Metrics data models for Pipecat framework.
|
||||||
|
|
||||||
|
This module defines Pydantic models for various types of metrics data
|
||||||
|
collected throughout the pipeline, including timing, token usage, and
|
||||||
|
processing statistics.
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
|
|
||||||
|
|
||||||
class MetricsData(BaseModel):
|
class MetricsData(BaseModel):
|
||||||
|
"""Base class for all metrics data.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
processor: Name of the processor generating the metrics.
|
||||||
|
model: Optional model name associated with the metrics.
|
||||||
|
"""
|
||||||
|
|
||||||
processor: str
|
processor: str
|
||||||
model: Optional[str] = None
|
model: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
class TTFBMetricsData(MetricsData):
|
class TTFBMetricsData(MetricsData):
|
||||||
|
"""Time To First Byte (TTFB) metrics data.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
value: TTFB measurement in seconds.
|
||||||
|
"""
|
||||||
|
|
||||||
value: float
|
value: float
|
||||||
|
|
||||||
|
|
||||||
class ProcessingMetricsData(MetricsData):
|
class ProcessingMetricsData(MetricsData):
|
||||||
|
"""General processing time metrics data.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
value: Processing time measurement in seconds.
|
||||||
|
"""
|
||||||
|
|
||||||
value: float
|
value: float
|
||||||
|
|
||||||
|
|
||||||
class LLMTokenUsage(BaseModel):
|
class LLMTokenUsage(BaseModel):
|
||||||
|
"""Token usage statistics for LLM operations.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
prompt_tokens: Number of tokens in the input prompt.
|
||||||
|
completion_tokens: Number of tokens in the generated completion.
|
||||||
|
total_tokens: Total number of tokens used (prompt + completion).
|
||||||
|
cache_read_input_tokens: Number of tokens read from cache, if applicable.
|
||||||
|
cache_creation_input_tokens: Number of tokens used to create cache entries, if applicable.
|
||||||
|
"""
|
||||||
|
|
||||||
prompt_tokens: int
|
prompt_tokens: int
|
||||||
completion_tokens: int
|
completion_tokens: int
|
||||||
total_tokens: int
|
total_tokens: int
|
||||||
@@ -26,15 +68,35 @@ class LLMTokenUsage(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class LLMUsageMetricsData(MetricsData):
|
class LLMUsageMetricsData(MetricsData):
|
||||||
|
"""LLM token usage metrics data.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
value: Token usage statistics for the LLM operation.
|
||||||
|
"""
|
||||||
|
|
||||||
value: LLMTokenUsage
|
value: LLMTokenUsage
|
||||||
|
|
||||||
|
|
||||||
class TTSUsageMetricsData(MetricsData):
|
class TTSUsageMetricsData(MetricsData):
|
||||||
|
"""Text-to-Speech usage metrics data.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
value: Number of characters processed by TTS.
|
||||||
|
"""
|
||||||
|
|
||||||
value: int
|
value: int
|
||||||
|
|
||||||
|
|
||||||
class SmartTurnMetricsData(MetricsData):
|
class SmartTurnMetricsData(MetricsData):
|
||||||
"""Metrics data for smart turn predictions."""
|
"""Metrics data for smart turn predictions.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
is_complete: Whether the turn is predicted to be complete.
|
||||||
|
probability: Confidence probability of the turn completion prediction.
|
||||||
|
inference_time_ms: Time taken for inference in milliseconds.
|
||||||
|
server_total_time_ms: Total server processing time in milliseconds.
|
||||||
|
e2e_processing_time_ms: End-to-end processing time in milliseconds.
|
||||||
|
"""
|
||||||
|
|
||||||
is_complete: bool
|
is_complete: bool
|
||||||
probability: float
|
probability: float
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base observer classes for monitoring frame flow in the Pipecat pipeline.
|
||||||
|
|
||||||
|
This module provides the foundation for observing frame transfers between
|
||||||
|
processors without modifying the pipeline structure. Observers can be used
|
||||||
|
for logging, debugging, analytics, and monitoring pipeline behavior.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
|
||||||
@@ -18,19 +25,19 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class FramePushed:
|
class FramePushed:
|
||||||
"""Represents an event where a frame is pushed from one processor to another
|
"""Event data for frame transfers between processors in the pipeline.
|
||||||
within the pipeline.
|
|
||||||
|
|
||||||
This data structure is typically used by observers to track the flow of
|
Represents an event where a frame is pushed from one processor to another
|
||||||
frames through the pipeline for logging, debugging, or analytics purposes.
|
within the pipeline. This data structure is typically used by observers
|
||||||
|
to track the flow of frames through the pipeline for logging, debugging,
|
||||||
Attributes:
|
or analytics purposes.
|
||||||
source (FrameProcessor): The processor sending the frame.
|
|
||||||
destination (FrameProcessor): The processor receiving the frame.
|
|
||||||
frame (Frame): The frame being transferred.
|
|
||||||
direction (FrameDirection): The direction of the transfer (e.g., downstream or upstream).
|
|
||||||
timestamp (int): The time when the frame was pushed, based on the pipeline clock.
|
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
source: The processor sending the frame.
|
||||||
|
destination: The processor receiving the frame.
|
||||||
|
frame: The frame being transferred.
|
||||||
|
direction: The direction of the transfer (e.g., downstream or upstream).
|
||||||
|
timestamp: The time when the frame was pushed, based on the pipeline clock.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
source: "FrameProcessor"
|
source: "FrameProcessor"
|
||||||
@@ -41,11 +48,12 @@ class FramePushed:
|
|||||||
|
|
||||||
|
|
||||||
class BaseObserver(BaseObject):
|
class BaseObserver(BaseObject):
|
||||||
"""This is the base class for pipeline frame observers. Observers can view
|
"""Base class for pipeline frame observers.
|
||||||
all the frames that go through the pipeline without the need to inject
|
|
||||||
processors in the pipeline. This can be useful, for example, to implement
|
|
||||||
frame loggers or debuggers among other things.
|
|
||||||
|
|
||||||
|
Observers can view all frames that flow through the pipeline without
|
||||||
|
needing to inject processors into the pipeline structure. This enables
|
||||||
|
non-intrusive monitoring capabilities such as frame logging, debugging,
|
||||||
|
performance analysis, and analytics collection.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
@@ -57,7 +65,6 @@ class BaseObserver(BaseObject):
|
|||||||
transferred through the pipeline.
|
transferred through the pipeline.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
data (FramePushed): The event data containing details about the frame transfer.
|
data: The event data containing details about the frame transfer.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Debug logging observer for frame activity monitoring.
|
||||||
|
|
||||||
|
This module provides a debug observer that logs detailed frame activity
|
||||||
|
to the console, making it useful for debugging pipeline behavior and
|
||||||
|
understanding frame flow between processors.
|
||||||
|
"""
|
||||||
|
|
||||||
from dataclasses import fields, is_dataclass
|
from dataclasses import fields, is_dataclass
|
||||||
from enum import Enum, auto
|
from enum import Enum, auto
|
||||||
from typing import Dict, Optional, Set, Tuple, Type, Union
|
from typing import Dict, Optional, Set, Tuple, Type, Union
|
||||||
@@ -16,7 +23,12 @@ from pipecat.processors.frame_processor import FrameDirection
|
|||||||
|
|
||||||
|
|
||||||
class FrameEndpoint(Enum):
|
class FrameEndpoint(Enum):
|
||||||
"""Specifies which endpoint (source or destination) to filter on."""
|
"""Specifies which endpoint (source or destination) to filter on.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
SOURCE: Filter on the source component that is pushing the frame.
|
||||||
|
DESTINATION: Filter on the destination component receiving the frame.
|
||||||
|
"""
|
||||||
|
|
||||||
SOURCE = auto()
|
SOURCE = auto()
|
||||||
DESTINATION = auto()
|
DESTINATION = auto()
|
||||||
@@ -28,44 +40,36 @@ class DebugLogObserver(BaseObserver):
|
|||||||
Automatically extracts and formats data from any frame type, making it useful
|
Automatically extracts and formats data from any frame type, making it useful
|
||||||
for debugging pipeline behavior without needing frame-specific observers.
|
for debugging pipeline behavior without needing frame-specific observers.
|
||||||
|
|
||||||
Args:
|
|
||||||
frame_types: Optional tuple of frame types to log, or a dict with frame type
|
|
||||||
filters. If None, logs all frame types.
|
|
||||||
exclude_fields: Optional set of field names to exclude from logging.
|
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
Log all frames from all services:
|
Log all frames from all services::
|
||||||
```python
|
|
||||||
observers = DebugLogObserver()
|
|
||||||
```
|
|
||||||
|
|
||||||
Log specific frame types from any source/destination:
|
observers = DebugLogObserver()
|
||||||
```python
|
|
||||||
from pipecat.frames.frames import TranscriptionFrame, InterimTranscriptionFrame
|
|
||||||
observers=[
|
|
||||||
DebugLogObserver(frame_types=(LLMTextFrame,TranscriptionFrame,)),
|
|
||||||
],
|
|
||||||
```
|
|
||||||
|
|
||||||
Log frames with specific source/destination filters:
|
Log specific frame types from any source/destination::
|
||||||
```python
|
|
||||||
from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
|
|
||||||
from pipecat.transports.base_output_transport import BaseOutputTransport
|
|
||||||
from pipecat.services.stt_service import STTService
|
|
||||||
|
|
||||||
observers=[
|
from pipecat.frames.frames import TranscriptionFrame, InterimTranscriptionFrame
|
||||||
DebugLogObserver(
|
observers=[
|
||||||
frame_types={
|
DebugLogObserver(frame_types=(LLMTextFrame,TranscriptionFrame,)),
|
||||||
# Only log StartInterruptionFrame when source is BaseOutputTransport
|
]
|
||||||
StartInterruptionFrame: (BaseOutputTransport, FrameEndpoint.SOURCE),
|
|
||||||
# Only log UserStartedSpeakingFrame when destination is STTService
|
Log frames with specific source/destination filters::
|
||||||
UserStartedSpeakingFrame: (STTService, FrameEndpoint.DESTINATION),
|
|
||||||
# Log LLMTextFrame regardless of source or destination type
|
from pipecat.frames.frames import StartInterruptionFrame, UserStartedSpeakingFrame, LLMTextFrame
|
||||||
LLMTextFrame: None,
|
from pipecat.transports.base_output_transport import BaseOutputTransport
|
||||||
}
|
from pipecat.services.stt_service import STTService
|
||||||
),
|
|
||||||
],
|
observers=[
|
||||||
```
|
DebugLogObserver(
|
||||||
|
frame_types={
|
||||||
|
# Only log StartInterruptionFrame when source is BaseOutputTransport
|
||||||
|
StartInterruptionFrame: (BaseOutputTransport, FrameEndpoint.SOURCE),
|
||||||
|
# Only log UserStartedSpeakingFrame when destination is STTService
|
||||||
|
UserStartedSpeakingFrame: (STTService, FrameEndpoint.DESTINATION),
|
||||||
|
# Log LLMTextFrame regardless of source or destination type
|
||||||
|
LLMTextFrame: None,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
]
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -79,14 +83,17 @@ class DebugLogObserver(BaseObserver):
|
|||||||
"""Initialize the debug log observer.
|
"""Initialize the debug log observer.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
frame_types: Tuple of frame types to log, or a dict mapping frame types to
|
frame_types: Frame types to log. Can be:
|
||||||
filter configurations. Filter configs can be:
|
|
||||||
- None to log all instances of the frame type
|
- Tuple of frame types to log all instances
|
||||||
- A tuple of (service_type, endpoint) to filter on a specific service
|
- Dict mapping frame types to filter configurations
|
||||||
and endpoint (SOURCE or DESTINATION)
|
- None to log all frames
|
||||||
If None is provided instead of a tuple/dict, log all frames.
|
|
||||||
exclude_fields: Set of field names to exclude from logging. If None, only binary
|
Filter configurations can be None (log all instances) or a tuple
|
||||||
data fields are excluded.
|
of (service_type, endpoint) to filter on specific services.
|
||||||
|
exclude_fields: Field names to exclude from logging. Defaults to
|
||||||
|
excluding binary data fields like 'audio', 'image', 'images'.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
"""
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
@@ -113,14 +120,7 @@ class DebugLogObserver(BaseObserver):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _format_value(self, value):
|
def _format_value(self, value):
|
||||||
"""Format a value for logging.
|
"""Format a value for logging."""
|
||||||
|
|
||||||
Args:
|
|
||||||
value: The value to format.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: A string representation of the value suitable for logging.
|
|
||||||
"""
|
|
||||||
if value is None:
|
if value is None:
|
||||||
return "None"
|
return "None"
|
||||||
elif isinstance(value, str):
|
elif isinstance(value, str):
|
||||||
@@ -143,16 +143,7 @@ class DebugLogObserver(BaseObserver):
|
|||||||
return str(value)
|
return str(value)
|
||||||
|
|
||||||
def _should_log_frame(self, frame, src, dst):
|
def _should_log_frame(self, frame, src, dst):
|
||||||
"""Determine if a frame should be logged based on filters.
|
"""Determine if a frame should be logged based on filters."""
|
||||||
|
|
||||||
Args:
|
|
||||||
frame: The frame being processed
|
|
||||||
src: The source component
|
|
||||||
dst: The destination component
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
bool: True if the frame should be logged, False otherwise
|
|
||||||
"""
|
|
||||||
# If no filters, log all frames
|
# If no filters, log all frames
|
||||||
if not self.frame_filters:
|
if not self.frame_filters:
|
||||||
return True
|
return True
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""LLM logging observer for Pipecat."""
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -34,10 +36,15 @@ class LLMLogObserver(BaseObserver):
|
|||||||
|
|
||||||
This allows you to track when the LLM starts responding, what it generates,
|
This allows you to track when the LLM starts responding, what it generates,
|
||||||
and when it finishes.
|
and when it finishes.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def on_push_frame(self, data: FramePushed):
|
async def on_push_frame(self, data: FramePushed):
|
||||||
|
"""Handle frame push events and log LLM-related activities.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: The frame push event data containing source, destination,
|
||||||
|
frame, direction, and timestamp information.
|
||||||
|
"""
|
||||||
src = data.source
|
src = data.source
|
||||||
dst = data.destination
|
dst = data.destination
|
||||||
frame = data.frame
|
frame = data.frame
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Transcription logging observer for Pipecat.
|
||||||
|
|
||||||
|
This module provides an observer that logs transcription frames to the console,
|
||||||
|
allowing developers to monitor speech-to-text activity in real-time.
|
||||||
|
"""
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
from pipecat.frames.frames import (
|
from pipecat.frames.frames import (
|
||||||
@@ -17,17 +23,23 @@ from pipecat.services.stt_service import STTService
|
|||||||
class TranscriptionLogObserver(BaseObserver):
|
class TranscriptionLogObserver(BaseObserver):
|
||||||
"""Observer to log transcription activity to the console.
|
"""Observer to log transcription activity to the console.
|
||||||
|
|
||||||
Logs all frame instances (only from STT service) of:
|
Monitors and logs all transcription frames from STT services, including
|
||||||
|
both final transcriptions and interim results. This allows developers
|
||||||
- TranscriptionFrame
|
to track speech recognition activity and debug transcription issues.
|
||||||
- InterimTranscriptionFrame
|
|
||||||
|
|
||||||
This allows you to track when the LLM starts responding, what it generates,
|
|
||||||
and when it finishes.
|
|
||||||
|
|
||||||
|
Only processes frames from STTService instances to avoid logging
|
||||||
|
unrelated transcription frames from other sources.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def on_push_frame(self, data: FramePushed):
|
async def on_push_frame(self, data: FramePushed):
|
||||||
|
"""Handle frame push events and log transcription frames.
|
||||||
|
|
||||||
|
Logs TranscriptionFrame and InterimTranscriptionFrame instances
|
||||||
|
with timestamps and user information for debugging purposes.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: Frame push event data containing source, frame, and timestamp.
|
||||||
|
"""
|
||||||
src = data.source
|
src = data.source
|
||||||
frame = data.frame
|
frame = data.frame
|
||||||
timestamp = data.timestamp
|
timestamp = data.timestamp
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Observer for measuring user-to-bot response latency."""
|
||||||
|
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
@@ -18,19 +20,28 @@ from pipecat.processors.frame_processor import FrameDirection
|
|||||||
|
|
||||||
|
|
||||||
class UserBotLatencyLogObserver(BaseObserver):
|
class UserBotLatencyLogObserver(BaseObserver):
|
||||||
"""Observer that logs the latency between when the user stops speaking and
|
"""Observer that measures time between user stopping speech and bot starting speech.
|
||||||
when the bot starts speaking.
|
|
||||||
|
|
||||||
This helps measure how quickly the AI services respond.
|
|
||||||
|
|
||||||
|
This helps measure how quickly the AI services respond by tracking
|
||||||
|
conversation turn timing and logging latency metrics.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
"""Initialize the latency observer.
|
||||||
|
|
||||||
|
Sets up tracking for processed frames and user speech timing
|
||||||
|
to calculate response latencies.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._processed_frames = set()
|
self._processed_frames = set()
|
||||||
self._user_stopped_time = 0
|
self._user_stopped_time = 0
|
||||||
|
|
||||||
async def on_push_frame(self, data: FramePushed):
|
async def on_push_frame(self, data: FramePushed):
|
||||||
|
"""Process frames to track speech timing and calculate latency.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: Frame push event containing the frame and direction information.
|
||||||
|
"""
|
||||||
# Only process downstream frames
|
# Only process downstream frames
|
||||||
if data.direction != FrameDirection.DOWNSTREAM:
|
if data.direction != FrameDirection.DOWNSTREAM:
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Turn tracking observer for conversation flow monitoring.
|
||||||
|
|
||||||
|
This module provides an observer that monitors conversation turns in a pipeline,
|
||||||
|
tracking when turns start and end based on user and bot speech patterns.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from collections import deque
|
from collections import deque
|
||||||
|
|
||||||
@@ -23,15 +29,30 @@ from pipecat.observers.base_observer import BaseObserver, FramePushed
|
|||||||
class TurnTrackingObserver(BaseObserver):
|
class TurnTrackingObserver(BaseObserver):
|
||||||
"""Observer that tracks conversation turns in a pipeline.
|
"""Observer that tracks conversation turns in a pipeline.
|
||||||
|
|
||||||
|
This observer monitors the flow of conversation by tracking when turns
|
||||||
|
start and end based on user and bot speaking patterns. It handles
|
||||||
|
interruptions, timeouts, and maintains turn state throughout the pipeline.
|
||||||
|
|
||||||
Turn tracking logic:
|
Turn tracking logic:
|
||||||
|
|
||||||
- The first turn starts immediately when the pipeline starts (StartFrame)
|
- The first turn starts immediately when the pipeline starts (StartFrame)
|
||||||
- Subsequent turns start when the user starts speaking
|
- Subsequent turns start when the user starts speaking
|
||||||
- A turn ends when the bot stops speaking and either:
|
- A turn ends when the bot stops speaking and either:
|
||||||
|
|
||||||
- The user starts speaking again
|
- The user starts speaking again
|
||||||
- A timeout period elapses with no more bot speech
|
- A timeout period elapses with no more bot speech
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, max_frames=100, turn_end_timeout_secs=2.5, **kwargs):
|
def __init__(self, max_frames=100, turn_end_timeout_secs=2.5, **kwargs):
|
||||||
|
"""Initialize the turn tracking observer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
max_frames: Maximum number of frame IDs to keep in history for
|
||||||
|
duplicate detection. Defaults to 100.
|
||||||
|
turn_end_timeout_secs: Timeout in seconds after bot stops speaking
|
||||||
|
before automatically ending the turn. Defaults to 2.5.
|
||||||
|
**kwargs: Additional arguments passed to the parent observer.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._turn_count = 0
|
self._turn_count = 0
|
||||||
self._is_turn_active = False
|
self._is_turn_active = False
|
||||||
@@ -49,7 +70,11 @@ class TurnTrackingObserver(BaseObserver):
|
|||||||
self._register_event_handler("on_turn_ended")
|
self._register_event_handler("on_turn_ended")
|
||||||
|
|
||||||
async def on_push_frame(self, data: FramePushed):
|
async def on_push_frame(self, data: FramePushed):
|
||||||
"""Process frame events for turn tracking."""
|
"""Process frame events for turn tracking.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: Frame push event data containing the frame and metadata.
|
||||||
|
"""
|
||||||
# Skip already processed frames
|
# Skip already processed frames
|
||||||
if data.frame.id in self._processed_frames:
|
if data.frame.id in self._processed_frames:
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base pipeline implementation for frame processing."""
|
||||||
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
@@ -11,9 +13,24 @@ from pipecat.processors.frame_processor import FrameProcessor
|
|||||||
|
|
||||||
|
|
||||||
class BasePipeline(FrameProcessor):
|
class BasePipeline(FrameProcessor):
|
||||||
|
"""Base class for all pipeline implementations.
|
||||||
|
|
||||||
|
Provides the foundation for pipeline processors that need to support
|
||||||
|
metrics collection from their contained processors.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
"""Initialize the base pipeline."""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def processors_with_metrics(self) -> List[FrameProcessor]:
|
def processors_with_metrics(self) -> List[FrameProcessor]:
|
||||||
|
"""Return processors that can generate metrics.
|
||||||
|
|
||||||
|
Implementing classes should collect and return all processors within
|
||||||
|
their pipeline that support metrics generation.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of frame processors that support metrics collection.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base pipeline task implementation for managing pipeline execution.
|
||||||
|
|
||||||
|
This module provides the abstract base class and configuration for pipeline
|
||||||
|
tasks that manage the lifecycle and execution of frame processing pipelines.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@@ -15,44 +21,81 @@ from pipecat.utils.base_object import BaseObject
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class PipelineTaskParams:
|
class PipelineTaskParams:
|
||||||
"""Specific configuration for the pipeline task."""
|
"""Configuration parameters for pipeline task execution.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
loop: The asyncio event loop to use for task execution.
|
||||||
|
"""
|
||||||
|
|
||||||
loop: asyncio.AbstractEventLoop
|
loop: asyncio.AbstractEventLoop
|
||||||
|
|
||||||
|
|
||||||
class BasePipelineTask(BaseObject):
|
class BasePipelineTask(BaseObject):
|
||||||
|
"""Abstract base class for pipeline task implementations.
|
||||||
|
|
||||||
|
Defines the interface for managing pipeline execution lifecycle,
|
||||||
|
including starting, stopping, and frame queuing operations.
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def has_finished(self) -> bool:
|
def has_finished(self) -> bool:
|
||||||
"""Indicates whether the tasks has finished. That is, all processors
|
"""Check if the pipeline task has finished execution.
|
||||||
have stopped.
|
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if all processors have stopped and the task is complete.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def stop_when_done(self):
|
async def stop_when_done(self):
|
||||||
"""This is a helper function that sends an EndFrame to the pipeline in
|
"""Schedule the pipeline to stop after processing all queued frames.
|
||||||
order to stop the task after everything in it has been processed.
|
|
||||||
|
|
||||||
|
Implementing classes should send an EndFrame or equivalent signal to
|
||||||
|
gracefully terminate the pipeline once all current processing is complete.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def cancel(self):
|
async def cancel(self):
|
||||||
"""Stops the running pipeline immediately."""
|
"""Immediately stop the running pipeline.
|
||||||
|
|
||||||
|
Implementing classes should cancel all running tasks and stop frame
|
||||||
|
processing without waiting for completion.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def run(self, params: PipelineTaskParams):
|
async def run(self, params: PipelineTaskParams):
|
||||||
"""Starts running the given pipeline."""
|
"""Start and run the pipeline with the given parameters.
|
||||||
|
|
||||||
|
Implementing classes should initialize and execute the pipeline using
|
||||||
|
the provided configuration parameters.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: Configuration parameters for pipeline execution.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def queue_frame(self, frame: Frame):
|
async def queue_frame(self, frame: Frame):
|
||||||
"""Queue a frame to be pushed down the pipeline."""
|
"""Queue a single frame for processing by the pipeline.
|
||||||
|
|
||||||
|
Implementing classes should add the frame to their processing queue
|
||||||
|
for downstream handling.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to be processed.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
|
async def queue_frames(self, frames: Iterable[Frame] | AsyncIterable[Frame]):
|
||||||
"""Queues multiple frames to be pushed down the pipeline."""
|
"""Queue multiple frames for processing by the pipeline.
|
||||||
|
|
||||||
|
Implementing classes should process the iterable/async iterable and
|
||||||
|
add all frames to their processing queue.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frames: An iterable or async iterable of frames to be processed.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Parallel pipeline implementation for concurrent frame processing.
|
||||||
|
|
||||||
|
This module provides a parallel pipeline that processes frames through multiple
|
||||||
|
sub-pipelines concurrently, with coordination for system frames and proper
|
||||||
|
handling of pipeline lifecycle events.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
from typing import Awaitable, Callable, Dict, List
|
from typing import Awaitable, Callable, Dict, List
|
||||||
@@ -25,16 +32,34 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
|
|||||||
|
|
||||||
|
|
||||||
class ParallelPipelineSource(FrameProcessor):
|
class ParallelPipelineSource(FrameProcessor):
|
||||||
|
"""Source processor for parallel pipeline branches.
|
||||||
|
|
||||||
|
Handles frame routing for parallel pipeline inputs, directing system frames
|
||||||
|
to the parent push function and other upstream frames to a queue for processing.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
upstream_queue: asyncio.Queue,
|
upstream_queue: asyncio.Queue,
|
||||||
push_frame_func: Callable[[Frame, FrameDirection], Awaitable[None]],
|
push_frame_func: Callable[[Frame, FrameDirection], Awaitable[None]],
|
||||||
):
|
):
|
||||||
|
"""Initialize the parallel pipeline source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
upstream_queue: Queue for collecting upstream frames from this branch.
|
||||||
|
push_frame_func: Function to push frames to the parent parallel pipeline.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._up_queue = upstream_queue
|
self._up_queue = upstream_queue
|
||||||
self._push_frame_func = push_frame_func
|
self._push_frame_func = push_frame_func
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames with special handling for system frames.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
match direction:
|
match direction:
|
||||||
@@ -48,16 +73,34 @@ class ParallelPipelineSource(FrameProcessor):
|
|||||||
|
|
||||||
|
|
||||||
class ParallelPipelineSink(FrameProcessor):
|
class ParallelPipelineSink(FrameProcessor):
|
||||||
|
"""Sink processor for parallel pipeline branches.
|
||||||
|
|
||||||
|
Handles frame routing for parallel pipeline outputs, directing system frames
|
||||||
|
to the parent push function and other downstream frames to a queue for coordination.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
downstream_queue: asyncio.Queue,
|
downstream_queue: asyncio.Queue,
|
||||||
push_frame_func: Callable[[Frame, FrameDirection], Awaitable[None]],
|
push_frame_func: Callable[[Frame, FrameDirection], Awaitable[None]],
|
||||||
):
|
):
|
||||||
|
"""Initialize the parallel pipeline sink.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
downstream_queue: Queue for collecting downstream frames from this branch.
|
||||||
|
push_frame_func: Function to push frames to the parent parallel pipeline.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._down_queue = downstream_queue
|
self._down_queue = downstream_queue
|
||||||
self._push_frame_func = push_frame_func
|
self._push_frame_func = push_frame_func
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames with special handling for system frames.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
match direction:
|
match direction:
|
||||||
@@ -71,7 +114,24 @@ class ParallelPipelineSink(FrameProcessor):
|
|||||||
|
|
||||||
|
|
||||||
class ParallelPipeline(BasePipeline):
|
class ParallelPipeline(BasePipeline):
|
||||||
|
"""Pipeline that processes frames through multiple sub-pipelines concurrently.
|
||||||
|
|
||||||
|
Creates multiple parallel processing branches from the provided processor lists,
|
||||||
|
coordinating frame flow and ensuring proper synchronization of lifecycle events
|
||||||
|
like EndFrames. Each branch runs independently while system frames are handled
|
||||||
|
specially to maintain pipeline coordination.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
|
"""Initialize the parallel pipeline with processor lists.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
*args: Variable number of processor lists, each becoming a parallel branch.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If no processor lists are provided.
|
||||||
|
TypeError: If any argument is not a list of processors.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
if len(args) == 0:
|
if len(args) == 0:
|
||||||
@@ -93,6 +153,11 @@ class ParallelPipeline(BasePipeline):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def processors_with_metrics(self) -> List[FrameProcessor]:
|
def processors_with_metrics(self) -> List[FrameProcessor]:
|
||||||
|
"""Collect processors that can generate metrics from all parallel branches.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of frame processors that support metrics collection from all branches.
|
||||||
|
"""
|
||||||
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
|
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
|
||||||
|
|
||||||
#
|
#
|
||||||
@@ -100,6 +165,14 @@ class ParallelPipeline(BasePipeline):
|
|||||||
#
|
#
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Set up the parallel pipeline and all its branches.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: Configuration for frame processor setup.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
TypeError: If any processor list argument is not actually a list.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
|
|
||||||
self._up_queue = WatchdogQueue(setup.task_manager)
|
self._up_queue = WatchdogQueue(setup.task_manager)
|
||||||
@@ -129,12 +202,19 @@ class ParallelPipeline(BasePipeline):
|
|||||||
await asyncio.gather(*[s.setup(setup) for s in self._sinks])
|
await asyncio.gather(*[s.setup(setup) for s in self._sinks])
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Clean up the parallel pipeline and all its branches."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await asyncio.gather(*[s.cleanup() for s in self._sources])
|
await asyncio.gather(*[s.cleanup() for s in self._sources])
|
||||||
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
|
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
|
||||||
await asyncio.gather(*[s.cleanup() for s in self._sinks])
|
await asyncio.gather(*[s.cleanup() for s in self._sinks])
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames through all parallel branches with lifecycle coordination.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
if isinstance(frame, StartFrame):
|
if isinstance(frame, StartFrame):
|
||||||
@@ -159,9 +239,11 @@ class ParallelPipeline(BasePipeline):
|
|||||||
await self._stop()
|
await self._stop()
|
||||||
|
|
||||||
async def _start(self, frame: StartFrame):
|
async def _start(self, frame: StartFrame):
|
||||||
|
"""Start the parallel pipeline processing tasks."""
|
||||||
await self._create_tasks()
|
await self._create_tasks()
|
||||||
|
|
||||||
async def _stop(self):
|
async def _stop(self):
|
||||||
|
"""Stop all parallel pipeline processing tasks."""
|
||||||
if self._up_task:
|
if self._up_task:
|
||||||
# The up task doesn't receive an EndFrame, so we just cancel it.
|
# The up task doesn't receive an EndFrame, so we just cancel it.
|
||||||
await self.cancel_task(self._up_task)
|
await self.cancel_task(self._up_task)
|
||||||
@@ -174,6 +256,7 @@ class ParallelPipeline(BasePipeline):
|
|||||||
self._down_task = None
|
self._down_task = None
|
||||||
|
|
||||||
async def _cancel(self):
|
async def _cancel(self):
|
||||||
|
"""Cancel all parallel pipeline processing tasks."""
|
||||||
if self._up_task:
|
if self._up_task:
|
||||||
await self.cancel_task(self._up_task)
|
await self.cancel_task(self._up_task)
|
||||||
self._up_task = None
|
self._up_task = None
|
||||||
@@ -182,34 +265,44 @@ class ParallelPipeline(BasePipeline):
|
|||||||
self._down_task = None
|
self._down_task = None
|
||||||
|
|
||||||
async def _create_tasks(self):
|
async def _create_tasks(self):
|
||||||
|
"""Create upstream and downstream processing tasks if not already running."""
|
||||||
if not self._up_task:
|
if not self._up_task:
|
||||||
self._up_task = self.create_task(self._process_up_queue())
|
self._up_task = self.create_task(self._process_up_queue())
|
||||||
if not self._down_task:
|
if not self._down_task:
|
||||||
self._down_task = self.create_task(self._process_down_queue())
|
self._down_task = self.create_task(self._process_down_queue())
|
||||||
|
|
||||||
async def _drain_queues(self):
|
async def _drain_queues(self):
|
||||||
|
"""Drain all frames from upstream and downstream queues."""
|
||||||
while not self._up_queue.empty:
|
while not self._up_queue.empty:
|
||||||
await self._up_queue.get()
|
await self._up_queue.get()
|
||||||
while not self._down_queue.empty:
|
while not self._down_queue.empty:
|
||||||
await self._down_queue.get()
|
await self._down_queue.get()
|
||||||
|
|
||||||
async def _handle_interruption(self):
|
async def _handle_interruption(self):
|
||||||
|
"""Handle interruption by cancelling tasks, draining queues, and restarting."""
|
||||||
await self._cancel()
|
await self._cancel()
|
||||||
await self._drain_queues()
|
await self._drain_queues()
|
||||||
await self._create_tasks()
|
await self._create_tasks()
|
||||||
|
|
||||||
async def _parallel_push_frame(self, frame: Frame, direction: FrameDirection):
|
async def _parallel_push_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Push frames while avoiding duplicates using frame ID tracking."""
|
||||||
if frame.id not in self._seen_ids:
|
if frame.id not in self._seen_ids:
|
||||||
self._seen_ids.add(frame.id)
|
self._seen_ids.add(frame.id)
|
||||||
await self.push_frame(frame, direction)
|
await self.push_frame(frame, direction)
|
||||||
|
|
||||||
async def _process_up_queue(self):
|
async def _process_up_queue(self):
|
||||||
|
"""Process upstream frames from all parallel branches."""
|
||||||
while True:
|
while True:
|
||||||
frame = await self._up_queue.get()
|
frame = await self._up_queue.get()
|
||||||
await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
|
await self._parallel_push_frame(frame, FrameDirection.UPSTREAM)
|
||||||
self._up_queue.task_done()
|
self._up_queue.task_done()
|
||||||
|
|
||||||
async def _process_down_queue(self):
|
async def _process_down_queue(self):
|
||||||
|
"""Process downstream frames with EndFrame coordination.
|
||||||
|
|
||||||
|
Coordinates EndFrames to ensure they are only pushed upstream once
|
||||||
|
all parallel branches have completed processing them.
|
||||||
|
"""
|
||||||
running = True
|
running = True
|
||||||
while running:
|
while running:
|
||||||
frame = await self._down_queue.get()
|
frame = await self._down_queue.get()
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Pipeline implementation for connecting and managing frame processors.
|
||||||
|
|
||||||
|
This module provides the main Pipeline class that connects frame processors
|
||||||
|
in sequence and manages frame flow between them, along with helper classes
|
||||||
|
for pipeline source and sink operations.
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import Callable, Coroutine, List
|
from typing import Callable, Coroutine, List
|
||||||
|
|
||||||
from pipecat.frames.frames import Frame
|
from pipecat.frames.frames import Frame
|
||||||
@@ -12,11 +19,29 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, F
|
|||||||
|
|
||||||
|
|
||||||
class PipelineSource(FrameProcessor):
|
class PipelineSource(FrameProcessor):
|
||||||
|
"""Source processor that forwards frames to an upstream handler.
|
||||||
|
|
||||||
|
This processor acts as the entry point for a pipeline, forwarding
|
||||||
|
downstream frames to the next processor and upstream frames to a
|
||||||
|
provided upstream handler function.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, upstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
|
def __init__(self, upstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
|
||||||
|
"""Initialize the pipeline source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
upstream_push_frame: Coroutine function to handle upstream frames.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._upstream_push_frame = upstream_push_frame
|
self._upstream_push_frame = upstream_push_frame
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and route them based on direction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
match direction:
|
match direction:
|
||||||
@@ -27,11 +52,29 @@ class PipelineSource(FrameProcessor):
|
|||||||
|
|
||||||
|
|
||||||
class PipelineSink(FrameProcessor):
|
class PipelineSink(FrameProcessor):
|
||||||
|
"""Sink processor that forwards frames to a downstream handler.
|
||||||
|
|
||||||
|
This processor acts as the exit point for a pipeline, forwarding
|
||||||
|
upstream frames to the previous processor and downstream frames to a
|
||||||
|
provided downstream handler function.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, downstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
|
def __init__(self, downstream_push_frame: Callable[[Frame, FrameDirection], Coroutine]):
|
||||||
|
"""Initialize the pipeline sink.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
downstream_push_frame: Coroutine function to handle downstream frames.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._downstream_push_frame = downstream_push_frame
|
self._downstream_push_frame = downstream_push_frame
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and route them based on direction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
match direction:
|
match direction:
|
||||||
@@ -42,7 +85,19 @@ class PipelineSink(FrameProcessor):
|
|||||||
|
|
||||||
|
|
||||||
class Pipeline(BasePipeline):
|
class Pipeline(BasePipeline):
|
||||||
|
"""Main pipeline implementation that connects frame processors in sequence.
|
||||||
|
|
||||||
|
Creates a linear chain of frame processors with automatic source and sink
|
||||||
|
processors for external frame handling. Manages processor lifecycle and
|
||||||
|
provides metrics collection from contained processors.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, processors: List[FrameProcessor]):
|
def __init__(self, processors: List[FrameProcessor]):
|
||||||
|
"""Initialize the pipeline with a list of processors.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
processors: List of frame processors to connect in sequence.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
# Add a source and a sink queue so we can forward frames upstream and
|
# Add a source and a sink queue so we can forward frames upstream and
|
||||||
@@ -58,6 +113,14 @@ class Pipeline(BasePipeline):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def processors_with_metrics(self):
|
def processors_with_metrics(self):
|
||||||
|
"""Return processors that can generate metrics.
|
||||||
|
|
||||||
|
Recursively collects all processors that support metrics generation,
|
||||||
|
including those from nested pipelines.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of frame processors that can generate metrics.
|
||||||
|
"""
|
||||||
services = []
|
services = []
|
||||||
for p in self._processors:
|
for p in self._processors:
|
||||||
if isinstance(p, BasePipeline):
|
if isinstance(p, BasePipeline):
|
||||||
@@ -71,14 +134,26 @@ class Pipeline(BasePipeline):
|
|||||||
#
|
#
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Set up the pipeline and all contained processors.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: Configuration for frame processor setup.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
await self._setup_processors(setup)
|
await self._setup_processors(setup)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Clean up the pipeline and all contained processors."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._cleanup_processors()
|
await self._cleanup_processors()
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames by routing them through the pipeline.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
if direction == FrameDirection.DOWNSTREAM:
|
if direction == FrameDirection.DOWNSTREAM:
|
||||||
@@ -87,14 +162,17 @@ class Pipeline(BasePipeline):
|
|||||||
await self._sink.queue_frame(frame, FrameDirection.UPSTREAM)
|
await self._sink.queue_frame(frame, FrameDirection.UPSTREAM)
|
||||||
|
|
||||||
async def _setup_processors(self, setup: FrameProcessorSetup):
|
async def _setup_processors(self, setup: FrameProcessorSetup):
|
||||||
|
"""Set up all processors in the pipeline."""
|
||||||
for p in self._processors:
|
for p in self._processors:
|
||||||
await p.setup(setup)
|
await p.setup(setup)
|
||||||
|
|
||||||
async def _cleanup_processors(self):
|
async def _cleanup_processors(self):
|
||||||
|
"""Clean up all processors in the pipeline."""
|
||||||
for p in self._processors:
|
for p in self._processors:
|
||||||
await p.cleanup()
|
await p.cleanup()
|
||||||
|
|
||||||
def _link_processors(self):
|
def _link_processors(self):
|
||||||
|
"""Link all processors in sequence and set their parent."""
|
||||||
prev = self._processors[0]
|
prev = self._processors[0]
|
||||||
for curr in self._processors[1:]:
|
for curr in self._processors[1:]:
|
||||||
prev.set_parent(self)
|
prev.set_parent(self)
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Pipeline runner for managing pipeline task execution.
|
||||||
|
|
||||||
|
This module provides the PipelineRunner class that handles the execution
|
||||||
|
of pipeline tasks with signal handling, garbage collection, and lifecycle
|
||||||
|
management.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import gc
|
import gc
|
||||||
import signal
|
import signal
|
||||||
@@ -17,6 +24,13 @@ from pipecat.utils.base_object import BaseObject
|
|||||||
|
|
||||||
|
|
||||||
class PipelineRunner(BaseObject):
|
class PipelineRunner(BaseObject):
|
||||||
|
"""Manages the execution of pipeline tasks with lifecycle and signal handling.
|
||||||
|
|
||||||
|
Provides a high-level interface for running pipeline tasks with automatic
|
||||||
|
signal handling (SIGINT/SIGTERM), optional garbage collection, and proper
|
||||||
|
cleanup of resources.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -25,6 +39,14 @@ class PipelineRunner(BaseObject):
|
|||||||
force_gc: bool = False,
|
force_gc: bool = False,
|
||||||
loop: Optional[asyncio.AbstractEventLoop] = None,
|
loop: Optional[asyncio.AbstractEventLoop] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the pipeline runner.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: Optional name for the runner instance.
|
||||||
|
handle_sigint: Whether to automatically handle SIGINT/SIGTERM signals.
|
||||||
|
force_gc: Whether to force garbage collection after task completion.
|
||||||
|
loop: Event loop to use. If None, uses the current running loop.
|
||||||
|
"""
|
||||||
super().__init__(name=name)
|
super().__init__(name=name)
|
||||||
|
|
||||||
self._tasks = {}
|
self._tasks = {}
|
||||||
@@ -36,6 +58,11 @@ class PipelineRunner(BaseObject):
|
|||||||
self._setup_sigint()
|
self._setup_sigint()
|
||||||
|
|
||||||
async def run(self, task: PipelineTask):
|
async def run(self, task: PipelineTask):
|
||||||
|
"""Run a pipeline task to completion.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task: The pipeline task to execute.
|
||||||
|
"""
|
||||||
logger.debug(f"Runner {self} started running {task}")
|
logger.debug(f"Runner {self} started running {task}")
|
||||||
self._tasks[task.name] = task
|
self._tasks[task.name] = task
|
||||||
params = PipelineTaskParams(loop=self._loop)
|
params = PipelineTaskParams(loop=self._loop)
|
||||||
@@ -56,27 +83,33 @@ class PipelineRunner(BaseObject):
|
|||||||
logger.debug(f"Runner {self} finished running {task}")
|
logger.debug(f"Runner {self} finished running {task}")
|
||||||
|
|
||||||
async def stop_when_done(self):
|
async def stop_when_done(self):
|
||||||
|
"""Schedule all running tasks to stop when their current processing is complete."""
|
||||||
logger.debug(f"Runner {self} scheduled to stop when all tasks are done")
|
logger.debug(f"Runner {self} scheduled to stop when all tasks are done")
|
||||||
await asyncio.gather(*[t.stop_when_done() for t in self._tasks.values()])
|
await asyncio.gather(*[t.stop_when_done() for t in self._tasks.values()])
|
||||||
|
|
||||||
async def cancel(self):
|
async def cancel(self):
|
||||||
|
"""Cancel all running tasks immediately."""
|
||||||
logger.debug(f"Cancelling runner {self}")
|
logger.debug(f"Cancelling runner {self}")
|
||||||
await asyncio.gather(*[t.cancel() for t in self._tasks.values()])
|
await asyncio.gather(*[t.cancel() for t in self._tasks.values()])
|
||||||
|
|
||||||
def _setup_sigint(self):
|
def _setup_sigint(self):
|
||||||
|
"""Set up signal handlers for graceful shutdown."""
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler())
|
loop.add_signal_handler(signal.SIGINT, lambda *args: self._sig_handler())
|
||||||
loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler())
|
loop.add_signal_handler(signal.SIGTERM, lambda *args: self._sig_handler())
|
||||||
|
|
||||||
def _sig_handler(self):
|
def _sig_handler(self):
|
||||||
|
"""Handle interrupt signals by cancelling all tasks."""
|
||||||
if not self._sig_task:
|
if not self._sig_task:
|
||||||
self._sig_task = asyncio.create_task(self._sig_cancel())
|
self._sig_task = asyncio.create_task(self._sig_cancel())
|
||||||
|
|
||||||
async def _sig_cancel(self):
|
async def _sig_cancel(self):
|
||||||
|
"""Cancel all running tasks due to signal interruption."""
|
||||||
logger.warning(f"Interruption detected. Cancelling runner {self}")
|
logger.warning(f"Interruption detected. Cancelling runner {self}")
|
||||||
await self.cancel()
|
await self.cancel()
|
||||||
|
|
||||||
def _gc_collect(self):
|
def _gc_collect(self):
|
||||||
|
"""Force garbage collection and log results."""
|
||||||
collected = gc.collect()
|
collected = gc.collect()
|
||||||
logger.debug(f"Garbage collector: collected {collected} objects.")
|
logger.debug(f"Garbage collector: collected {collected} objects.")
|
||||||
logger.debug(f"Garbage collector: uncollectable objects {gc.garbage}")
|
logger.debug(f"Garbage collector: uncollectable objects {gc.garbage}")
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Synchronous parallel pipeline implementation for concurrent frame processing.
|
||||||
|
|
||||||
|
This module provides a pipeline that processes frames through multiple parallel
|
||||||
|
pipelines simultaneously, synchronizing their output to maintain frame ordering
|
||||||
|
and prevent duplicate processing.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from itertools import chain
|
from itertools import chain
|
||||||
@@ -20,17 +27,38 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SyncFrame(ControlFrame):
|
class SyncFrame(ControlFrame):
|
||||||
"""This frame is used to know when the internal pipelines have finished."""
|
"""Control frame used to synchronize parallel pipeline processing.
|
||||||
|
|
||||||
|
This frame is sent through parallel pipelines to determine when the
|
||||||
|
internal pipelines have finished processing a batch of frames.
|
||||||
|
"""
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class SyncParallelPipelineSource(FrameProcessor):
|
class SyncParallelPipelineSource(FrameProcessor):
|
||||||
|
"""Source processor for synchronous parallel pipeline processing.
|
||||||
|
|
||||||
|
Routes frames to parallel pipelines and collects upstream responses
|
||||||
|
for synchronization purposes.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, upstream_queue: asyncio.Queue):
|
def __init__(self, upstream_queue: asyncio.Queue):
|
||||||
|
"""Initialize the sync parallel pipeline source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
upstream_queue: Queue for collecting upstream frames from the pipeline.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._up_queue = upstream_queue
|
self._up_queue = upstream_queue
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and route them based on direction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
match direction:
|
match direction:
|
||||||
@@ -41,11 +69,28 @@ class SyncParallelPipelineSource(FrameProcessor):
|
|||||||
|
|
||||||
|
|
||||||
class SyncParallelPipelineSink(FrameProcessor):
|
class SyncParallelPipelineSink(FrameProcessor):
|
||||||
|
"""Sink processor for synchronous parallel pipeline processing.
|
||||||
|
|
||||||
|
Collects downstream frames from parallel pipelines and routes
|
||||||
|
upstream frames back through the pipeline.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, downstream_queue: asyncio.Queue):
|
def __init__(self, downstream_queue: asyncio.Queue):
|
||||||
|
"""Initialize the sync parallel pipeline sink.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
downstream_queue: Queue for collecting downstream frames from the pipeline.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._down_queue = downstream_queue
|
self._down_queue = downstream_queue
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and route them based on direction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
match direction:
|
match direction:
|
||||||
@@ -56,7 +101,28 @@ class SyncParallelPipelineSink(FrameProcessor):
|
|||||||
|
|
||||||
|
|
||||||
class SyncParallelPipeline(BasePipeline):
|
class SyncParallelPipeline(BasePipeline):
|
||||||
|
"""Pipeline that processes frames through multiple parallel pipelines synchronously.
|
||||||
|
|
||||||
|
Creates multiple parallel processing paths that all receive the same input frames
|
||||||
|
and produces synchronized output. Each parallel path is a separate pipeline that
|
||||||
|
processes frames independently, with synchronization points to ensure consistent
|
||||||
|
ordering and prevent duplicate frame processing.
|
||||||
|
|
||||||
|
The pipeline uses SyncFrame control frames to coordinate between parallel paths
|
||||||
|
and ensure all paths have completed processing before moving to the next frame.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, *args):
|
def __init__(self, *args):
|
||||||
|
"""Initialize the synchronous parallel pipeline.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
*args: Variable number of processor lists, each representing a parallel pipeline path.
|
||||||
|
Each argument should be a list of FrameProcessor instances.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If no arguments are provided.
|
||||||
|
TypeError: If any argument is not a list of processors.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
if len(args) == 0:
|
if len(args) == 0:
|
||||||
@@ -72,6 +138,11 @@ class SyncParallelPipeline(BasePipeline):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def processors_with_metrics(self) -> List[FrameProcessor]:
|
def processors_with_metrics(self) -> List[FrameProcessor]:
|
||||||
|
"""Collect processors that can generate metrics from all parallel pipelines.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of frame processors that support metrics collection from all parallel paths.
|
||||||
|
"""
|
||||||
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
|
return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines))
|
||||||
|
|
||||||
#
|
#
|
||||||
@@ -79,6 +150,11 @@ class SyncParallelPipeline(BasePipeline):
|
|||||||
#
|
#
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Set up the parallel pipeline and all contained processors.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: Configuration for frame processor setup.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
|
|
||||||
self._up_queue = WatchdogQueue(setup.task_manager)
|
self._up_queue = WatchdogQueue(setup.task_manager)
|
||||||
@@ -113,12 +189,23 @@ class SyncParallelPipeline(BasePipeline):
|
|||||||
await asyncio.gather(*[s["processor"].setup(setup) for s in self._sinks])
|
await asyncio.gather(*[s["processor"].setup(setup) for s in self._sinks])
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Clean up the parallel pipeline and all contained processors."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await asyncio.gather(*[s["processor"].cleanup() for s in self._sources])
|
await asyncio.gather(*[s["processor"].cleanup() for s in self._sources])
|
||||||
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
|
await asyncio.gather(*[p.cleanup() for p in self._pipelines])
|
||||||
await asyncio.gather(*[s["processor"].cleanup() for s in self._sinks])
|
await asyncio.gather(*[s["processor"].cleanup() for s in self._sinks])
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames through all parallel pipelines with synchronization.
|
||||||
|
|
||||||
|
Distributes frames to all parallel pipelines and synchronizes their output
|
||||||
|
to maintain proper ordering and prevent duplicate processing. Uses SyncFrame
|
||||||
|
control frames to coordinate between parallel paths.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
# The last processor of each pipeline needs to be synchronous otherwise
|
# The last processor of each pipeline needs to be synchronous otherwise
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Pipeline task implementation for managing frame processing pipelines.
|
||||||
|
|
||||||
|
This module provides the main PipelineTask class that orchestrates pipeline
|
||||||
|
execution, frame routing, lifecycle management, and monitoring capabilities
|
||||||
|
including heartbeats, idle detection, and observer integration.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import time
|
import time
|
||||||
from collections import deque
|
from collections import deque
|
||||||
@@ -53,12 +60,13 @@ HEARTBEAT_MONITOR_SECONDS = HEARTBEAT_SECONDS * 10
|
|||||||
|
|
||||||
|
|
||||||
class PipelineParams(BaseModel):
|
class PipelineParams(BaseModel):
|
||||||
"""Configuration parameters for pipeline execution. These parameters are
|
"""Configuration parameters for pipeline execution.
|
||||||
usually passed to all frame processors using through `StartFrame`. For other
|
|
||||||
generic pipeline task parameters use `PipelineTask` constructor arguments
|
|
||||||
instead.
|
|
||||||
|
|
||||||
Attributes:
|
These parameters are usually passed to all frame processors through
|
||||||
|
StartFrame. For other generic pipeline task parameters use PipelineTask
|
||||||
|
constructor arguments instead.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
allow_interruptions: Whether to allow pipeline interruptions.
|
allow_interruptions: Whether to allow pipeline interruptions.
|
||||||
audio_in_sample_rate: Input audio sample rate in Hz.
|
audio_in_sample_rate: Input audio sample rate in Hz.
|
||||||
audio_out_sample_rate: Output audio sample rate in Hz.
|
audio_out_sample_rate: Output audio sample rate in Hz.
|
||||||
@@ -66,12 +74,11 @@ class PipelineParams(BaseModel):
|
|||||||
enable_metrics: Whether to enable metrics collection.
|
enable_metrics: Whether to enable metrics collection.
|
||||||
enable_usage_metrics: Whether to enable usage metrics.
|
enable_usage_metrics: Whether to enable usage metrics.
|
||||||
heartbeats_period_secs: Period between heartbeats in seconds.
|
heartbeats_period_secs: Period between heartbeats in seconds.
|
||||||
|
interruption_strategies: Strategies for bot interruption behavior.
|
||||||
observers: [deprecated] Use `observers` arg in `PipelineTask` class.
|
observers: [deprecated] Use `observers` arg in `PipelineTask` class.
|
||||||
report_only_initial_ttfb: Whether to report only initial time to first byte.
|
report_only_initial_ttfb: Whether to report only initial time to first byte.
|
||||||
send_initial_empty_metrics: Whether to send initial empty metrics.
|
send_initial_empty_metrics: Whether to send initial empty metrics.
|
||||||
start_metadata: Additional metadata for pipeline start.
|
start_metadata: Additional metadata for pipeline start.
|
||||||
interruption_strategies: Strategies for bot interruption behavior.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||||
@@ -97,17 +104,25 @@ class PipelineTaskSource(FrameProcessor):
|
|||||||
pipeline given to the pipeline task. It allows us to easily push frames
|
pipeline given to the pipeline task. It allows us to easily push frames
|
||||||
downstream to the pipeline and also receive upstream frames coming from the
|
downstream to the pipeline and also receive upstream frames coming from the
|
||||||
pipeline.
|
pipeline.
|
||||||
|
|
||||||
Args:
|
|
||||||
up_queue: Queue for upstream frame processing.
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, up_queue: asyncio.Queue, **kwargs):
|
def __init__(self, up_queue: asyncio.Queue, **kwargs):
|
||||||
|
"""Initialize the pipeline task source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
up_queue: Queue for upstream frame processing.
|
||||||
|
**kwargs: Additional arguments passed to the parent class.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._up_queue = up_queue
|
self._up_queue = up_queue
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and route them based on direction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
match direction:
|
match direction:
|
||||||
@@ -123,16 +138,25 @@ class PipelineTaskSink(FrameProcessor):
|
|||||||
This is the sink processor that is linked at the end of the pipeline
|
This is the sink processor that is linked at the end of the pipeline
|
||||||
given to the pipeline task. It allows us to receive downstream frames and
|
given to the pipeline task. It allows us to receive downstream frames and
|
||||||
act on them, for example, waiting to receive an EndFrame.
|
act on them, for example, waiting to receive an EndFrame.
|
||||||
|
|
||||||
Args:
|
|
||||||
down_queue: Queue for downstream frame processing.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, down_queue: asyncio.Queue, **kwargs):
|
def __init__(self, down_queue: asyncio.Queue, **kwargs):
|
||||||
|
"""Initialize the pipeline task sink.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
down_queue: Queue for downstream frame processing.
|
||||||
|
**kwargs: Additional arguments passed to the parent class.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._down_queue = down_queue
|
self._down_queue = down_queue
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and route them to the downstream queue.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
await self._down_queue.put(frame)
|
await self._down_queue.put(frame)
|
||||||
|
|
||||||
@@ -140,69 +164,30 @@ class PipelineTaskSink(FrameProcessor):
|
|||||||
class PipelineTask(BasePipelineTask):
|
class PipelineTask(BasePipelineTask):
|
||||||
"""Manages the execution of a pipeline, handling frame processing and task lifecycle.
|
"""Manages the execution of a pipeline, handling frame processing and task lifecycle.
|
||||||
|
|
||||||
It has a couple of event handlers `on_frame_reached_upstream` and
|
This class orchestrates pipeline execution with comprehensive monitoring,
|
||||||
`on_frame_reached_downstream` that are called when upstream frames or
|
event handling, and lifecycle management. It provides event handlers for
|
||||||
downstream frames reach both ends of pipeline. By default, the events
|
various pipeline states and frame types, idle detection, heartbeat monitoring,
|
||||||
handlers will not be called unless some filters are set using
|
and observer integration.
|
||||||
`set_reached_upstream_filter` and `set_reached_downstream_filter`.
|
|
||||||
|
|
||||||
@task.event_handler("on_frame_reached_upstream")
|
Event handlers available:
|
||||||
async def on_frame_reached_upstream(task, frame):
|
|
||||||
...
|
|
||||||
|
|
||||||
@task.event_handler("on_frame_reached_downstream")
|
- on_frame_reached_upstream: Called when upstream frames reach the source
|
||||||
async def on_frame_reached_downstream(task, frame):
|
- on_frame_reached_downstream: Called when downstream frames reach the sink
|
||||||
...
|
- on_idle_timeout: Called when pipeline is idle beyond timeout threshold
|
||||||
|
- on_pipeline_started: Called when pipeline starts with StartFrame
|
||||||
|
- on_pipeline_stopped: Called when pipeline stops with StopFrame
|
||||||
|
- on_pipeline_ended: Called when pipeline ends with EndFrame
|
||||||
|
- on_pipeline_cancelled: Called when pipeline is cancelled
|
||||||
|
|
||||||
It also has an event handler that detects when the pipeline is idle. By
|
Example::
|
||||||
default, a pipeline is idle if no `BotSpeakingFrame` or
|
|
||||||
`LLMFullResponseEndFrame` are received within `idle_timeout_secs`.
|
|
||||||
|
|
||||||
@task.event_handler("on_idle_timeout")
|
@task.event_handler("on_frame_reached_upstream")
|
||||||
async def on_pipeline_idle_timeout(task):
|
async def on_frame_reached_upstream(task, frame):
|
||||||
...
|
...
|
||||||
|
|
||||||
There are also events to know if a pipeline has been started, stopped, ended
|
@task.event_handler("on_idle_timeout")
|
||||||
or cancelled.
|
async def on_pipeline_idle_timeout(task):
|
||||||
|
...
|
||||||
@task.event_handler("on_pipeline_started")
|
|
||||||
async def on_pipeline_started(task, frame: StartFrame):
|
|
||||||
...
|
|
||||||
|
|
||||||
@task.event_handler("on_pipeline_stopped")
|
|
||||||
async def on_pipeline_stopped(task, frame: StopFrame):
|
|
||||||
...
|
|
||||||
|
|
||||||
@task.event_handler("on_pipeline_ended")
|
|
||||||
async def on_pipeline_ended(task, frame: EndFrame):
|
|
||||||
...
|
|
||||||
|
|
||||||
@task.event_handler("on_pipeline_cancelled")
|
|
||||||
async def on_pipeline_cancelled(task, frame: CancelFrame):
|
|
||||||
...
|
|
||||||
|
|
||||||
Args:
|
|
||||||
pipeline: The pipeline to execute.
|
|
||||||
params: Configuration parameters for the pipeline.
|
|
||||||
additional_span_attributes: Optional dictionary of attributes to propagate as
|
|
||||||
OpenTelemetry conversation span attributes.
|
|
||||||
cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
|
|
||||||
the idle timeout is reached.
|
|
||||||
check_dangling_tasks: Whether to check for processors' tasks finishing properly.
|
|
||||||
clock: Clock implementation for timing operations.
|
|
||||||
conversation_id: Optional custom ID for the conversation.
|
|
||||||
enable_tracing: Whether to enable tracing.
|
|
||||||
enable_turn_tracking: Whether to enable turn tracking.
|
|
||||||
enable_watchdog_logging: Whether to print task processing times.
|
|
||||||
enable_watchdog_timers: Whether to enable task watchdog timers.
|
|
||||||
idle_timeout_frames: A tuple with the frames that should trigger an idle
|
|
||||||
timeout if not received withing `idle_timeout_seconds`.
|
|
||||||
idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or
|
|
||||||
None. If a pipeline is idle the pipeline task will be cancelled
|
|
||||||
automatically.
|
|
||||||
observers: List of observers for monitoring pipeline execution.
|
|
||||||
watchdog_timeout_secs: Watchdog timer timeout (in seconds). A warning
|
|
||||||
will be logged if the watchdog timer is not reset before this timeout.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -228,6 +213,32 @@ class PipelineTask(BasePipelineTask):
|
|||||||
task_manager: Optional[BaseTaskManager] = None,
|
task_manager: Optional[BaseTaskManager] = None,
|
||||||
watchdog_timeout_secs: float = WATCHDOG_TIMEOUT,
|
watchdog_timeout_secs: float = WATCHDOG_TIMEOUT,
|
||||||
):
|
):
|
||||||
|
"""Initialize the PipelineTask.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pipeline: The pipeline to execute.
|
||||||
|
params: Configuration parameters for the pipeline.
|
||||||
|
additional_span_attributes: Optional dictionary of attributes to propagate as
|
||||||
|
OpenTelemetry conversation span attributes.
|
||||||
|
cancel_on_idle_timeout: Whether the pipeline task should be cancelled if
|
||||||
|
the idle timeout is reached.
|
||||||
|
check_dangling_tasks: Whether to check for processors' tasks finishing properly.
|
||||||
|
clock: Clock implementation for timing operations.
|
||||||
|
conversation_id: Optional custom ID for the conversation.
|
||||||
|
enable_tracing: Whether to enable tracing.
|
||||||
|
enable_turn_tracking: Whether to enable turn tracking.
|
||||||
|
enable_watchdog_logging: Whether to print task processing times.
|
||||||
|
enable_watchdog_timers: Whether to enable task watchdog timers.
|
||||||
|
idle_timeout_frames: A tuple with the frames that should trigger an idle
|
||||||
|
timeout if not received within `idle_timeout_seconds`.
|
||||||
|
idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or
|
||||||
|
None. If a pipeline is idle the pipeline task will be cancelled
|
||||||
|
automatically.
|
||||||
|
observers: List of observers for monitoring pipeline execution.
|
||||||
|
task_manager: Optional task manager for handling asyncio tasks.
|
||||||
|
watchdog_timeout_secs: Watchdog timer timeout (in seconds). A warning
|
||||||
|
will be logged if the watchdog timer is not reset before this timeout.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._pipeline = pipeline
|
self._pipeline = pipeline
|
||||||
self._params = params or PipelineParams()
|
self._params = params or PipelineParams()
|
||||||
@@ -331,60 +342,97 @@ class PipelineTask(BasePipelineTask):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def params(self) -> PipelineParams:
|
def params(self) -> PipelineParams:
|
||||||
"""Returns the pipeline parameters of this task."""
|
"""Get the pipeline parameters for this task.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The pipeline parameters configuration.
|
||||||
|
"""
|
||||||
return self._params
|
return self._params
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def turn_tracking_observer(self) -> Optional[TurnTrackingObserver]:
|
def turn_tracking_observer(self) -> Optional[TurnTrackingObserver]:
|
||||||
"""Return the turn tracking observer if enabled."""
|
"""Get the turn tracking observer if enabled.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The turn tracking observer instance or None if not enabled.
|
||||||
|
"""
|
||||||
return self._turn_tracking_observer
|
return self._turn_tracking_observer
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def turn_trace_observer(self) -> Optional[TurnTraceObserver]:
|
def turn_trace_observer(self) -> Optional[TurnTraceObserver]:
|
||||||
"""Return the turn trace observer if enabled."""
|
"""Get the turn trace observer if enabled.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The turn trace observer instance or None if not enabled.
|
||||||
|
"""
|
||||||
return self._turn_trace_observer
|
return self._turn_trace_observer
|
||||||
|
|
||||||
def add_observer(self, observer: BaseObserver):
|
def add_observer(self, observer: BaseObserver):
|
||||||
|
"""Add an observer to monitor pipeline execution.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
observer: The observer to add to the pipeline monitoring.
|
||||||
|
"""
|
||||||
self._observer.add_observer(observer)
|
self._observer.add_observer(observer)
|
||||||
|
|
||||||
async def remove_observer(self, observer: BaseObserver):
|
async def remove_observer(self, observer: BaseObserver):
|
||||||
|
"""Remove an observer from pipeline monitoring.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
observer: The observer to remove from pipeline monitoring.
|
||||||
|
"""
|
||||||
await self._observer.remove_observer(observer)
|
await self._observer.remove_observer(observer)
|
||||||
|
|
||||||
def set_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]):
|
def set_reached_upstream_filter(self, types: Tuple[Type[Frame], ...]):
|
||||||
"""Sets which frames will be checked before calling the
|
"""Set which frame types trigger the on_frame_reached_upstream event.
|
||||||
on_frame_reached_upstream event handler.
|
|
||||||
|
|
||||||
|
Args:
|
||||||
|
types: Tuple of frame types to monitor for upstream events.
|
||||||
"""
|
"""
|
||||||
self._reached_upstream_types = types
|
self._reached_upstream_types = types
|
||||||
|
|
||||||
def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
|
def set_reached_downstream_filter(self, types: Tuple[Type[Frame], ...]):
|
||||||
"""Sets which frames will be checked before calling the
|
"""Set which frame types trigger the on_frame_reached_downstream event.
|
||||||
on_frame_reached_downstream event handler.
|
|
||||||
|
|
||||||
|
Args:
|
||||||
|
types: Tuple of frame types to monitor for downstream events.
|
||||||
"""
|
"""
|
||||||
self._reached_downstream_types = types
|
self._reached_downstream_types = types
|
||||||
|
|
||||||
def has_finished(self) -> bool:
|
def has_finished(self) -> bool:
|
||||||
"""Indicates whether the tasks has finished. That is, all processors
|
"""Check if the pipeline task has finished execution.
|
||||||
|
|
||||||
|
This indicates whether the tasks has finished, meaninig all processors
|
||||||
have stopped.
|
have stopped.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if all processors have stopped and the task is complete.
|
||||||
"""
|
"""
|
||||||
return self._finished
|
return self._finished
|
||||||
|
|
||||||
async def stop_when_done(self):
|
async def stop_when_done(self):
|
||||||
"""This is a helper function that sends an EndFrame to the pipeline in
|
"""Schedule the pipeline to stop after processing all queued frames.
|
||||||
order to stop the task after everything in it has been processed.
|
|
||||||
|
|
||||||
|
Sends an EndFrame to gracefully terminate the pipeline once all
|
||||||
|
current processing is complete.
|
||||||
"""
|
"""
|
||||||
logger.debug(f"Task {self} scheduled to stop when done")
|
logger.debug(f"Task {self} scheduled to stop when done")
|
||||||
await self.queue_frame(EndFrame())
|
await self.queue_frame(EndFrame())
|
||||||
|
|
||||||
async def cancel(self):
|
async def cancel(self):
|
||||||
"""Stops the running pipeline immediately."""
|
"""Immediately stop the running pipeline.
|
||||||
|
|
||||||
|
Cancels all running tasks and stops frame processing without
|
||||||
|
waiting for completion.
|
||||||
|
"""
|
||||||
await self._cancel()
|
await self._cancel()
|
||||||
|
|
||||||
async def run(self, params: PipelineTaskParams):
|
async def run(self, params: PipelineTaskParams):
|
||||||
"""Starts and manages the pipeline execution until completion or cancellation."""
|
"""Start and manage the pipeline execution until completion or cancellation.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: Configuration parameters for pipeline execution.
|
||||||
|
"""
|
||||||
if self.has_finished():
|
if self.has_finished():
|
||||||
return
|
return
|
||||||
cleanup_pipeline = True
|
cleanup_pipeline = True
|
||||||
@@ -440,6 +488,7 @@ class PipelineTask(BasePipelineTask):
|
|||||||
await self.queue_frame(frame)
|
await self.queue_frame(frame)
|
||||||
|
|
||||||
async def _cancel(self):
|
async def _cancel(self):
|
||||||
|
"""Internal cancellation logic for the pipeline task."""
|
||||||
if not self._cancelled:
|
if not self._cancelled:
|
||||||
logger.debug(f"Canceling pipeline task {self}")
|
logger.debug(f"Canceling pipeline task {self}")
|
||||||
self._cancelled = True
|
self._cancelled = True
|
||||||
@@ -453,6 +502,7 @@ class PipelineTask(BasePipelineTask):
|
|||||||
self._process_push_task = None
|
self._process_push_task = None
|
||||||
|
|
||||||
async def _create_tasks(self):
|
async def _create_tasks(self):
|
||||||
|
"""Create and start all pipeline processing tasks."""
|
||||||
self._process_up_task = self._task_manager.create_task(
|
self._process_up_task = self._task_manager.create_task(
|
||||||
self._process_up_queue(), f"{self}::_process_up_queue"
|
self._process_up_queue(), f"{self}::_process_up_queue"
|
||||||
)
|
)
|
||||||
@@ -468,6 +518,7 @@ class PipelineTask(BasePipelineTask):
|
|||||||
return self._process_push_task
|
return self._process_push_task
|
||||||
|
|
||||||
def _maybe_start_heartbeat_tasks(self):
|
def _maybe_start_heartbeat_tasks(self):
|
||||||
|
"""Start heartbeat tasks if heartbeats are enabled and not already running."""
|
||||||
if self._params.enable_heartbeats and self._heartbeat_push_task is None:
|
if self._params.enable_heartbeats and self._heartbeat_push_task is None:
|
||||||
self._heartbeat_push_task = self._task_manager.create_task(
|
self._heartbeat_push_task = self._task_manager.create_task(
|
||||||
self._heartbeat_push_handler(), f"{self}::_heartbeat_push_handler"
|
self._heartbeat_push_handler(), f"{self}::_heartbeat_push_handler"
|
||||||
@@ -477,12 +528,14 @@ class PipelineTask(BasePipelineTask):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _maybe_start_idle_task(self):
|
def _maybe_start_idle_task(self):
|
||||||
|
"""Start idle monitoring task if idle timeout is configured."""
|
||||||
if self._idle_timeout_secs:
|
if self._idle_timeout_secs:
|
||||||
self._idle_monitor_task = self._task_manager.create_task(
|
self._idle_monitor_task = self._task_manager.create_task(
|
||||||
self._idle_monitor_handler(), f"{self}::_idle_monitor_handler"
|
self._idle_monitor_handler(), f"{self}::_idle_monitor_handler"
|
||||||
)
|
)
|
||||||
|
|
||||||
async def _cancel_tasks(self):
|
async def _cancel_tasks(self):
|
||||||
|
"""Cancel all running pipeline tasks."""
|
||||||
await self._observer.stop()
|
await self._observer.stop()
|
||||||
|
|
||||||
if self._process_up_task:
|
if self._process_up_task:
|
||||||
@@ -497,6 +550,7 @@ class PipelineTask(BasePipelineTask):
|
|||||||
await self._maybe_cancel_idle_task()
|
await self._maybe_cancel_idle_task()
|
||||||
|
|
||||||
async def _maybe_cancel_heartbeat_tasks(self):
|
async def _maybe_cancel_heartbeat_tasks(self):
|
||||||
|
"""Cancel heartbeat tasks if they are running."""
|
||||||
if not self._params.enable_heartbeats:
|
if not self._params.enable_heartbeats:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -509,11 +563,13 @@ class PipelineTask(BasePipelineTask):
|
|||||||
self._heartbeat_monitor_task = None
|
self._heartbeat_monitor_task = None
|
||||||
|
|
||||||
async def _maybe_cancel_idle_task(self):
|
async def _maybe_cancel_idle_task(self):
|
||||||
|
"""Cancel idle monitoring task if it is running."""
|
||||||
if self._idle_timeout_secs and self._idle_monitor_task:
|
if self._idle_timeout_secs and self._idle_monitor_task:
|
||||||
await self._task_manager.cancel_task(self._idle_monitor_task)
|
await self._task_manager.cancel_task(self._idle_monitor_task)
|
||||||
self._idle_monitor_task = None
|
self._idle_monitor_task = None
|
||||||
|
|
||||||
def _initial_metrics_frame(self) -> MetricsFrame:
|
def _initial_metrics_frame(self) -> MetricsFrame:
|
||||||
|
"""Create an initial metrics frame with zero values for all processors."""
|
||||||
processors = self._pipeline.processors_with_metrics()
|
processors = self._pipeline.processors_with_metrics()
|
||||||
data = []
|
data = []
|
||||||
for p in processors:
|
for p in processors:
|
||||||
@@ -522,10 +578,12 @@ class PipelineTask(BasePipelineTask):
|
|||||||
return MetricsFrame(data=data)
|
return MetricsFrame(data=data)
|
||||||
|
|
||||||
async def _wait_for_pipeline_end(self):
|
async def _wait_for_pipeline_end(self):
|
||||||
|
"""Wait for the pipeline to signal completion."""
|
||||||
await self._pipeline_end_event.wait()
|
await self._pipeline_end_event.wait()
|
||||||
self._pipeline_end_event.clear()
|
self._pipeline_end_event.clear()
|
||||||
|
|
||||||
async def _setup(self, params: PipelineTaskParams):
|
async def _setup(self, params: PipelineTaskParams):
|
||||||
|
"""Set up the pipeline task and all processors."""
|
||||||
mgr_params = TaskManagerParams(
|
mgr_params = TaskManagerParams(
|
||||||
loop=params.loop,
|
loop=params.loop,
|
||||||
enable_watchdog_logging=self._enable_watchdog_logging,
|
enable_watchdog_logging=self._enable_watchdog_logging,
|
||||||
@@ -545,6 +603,7 @@ class PipelineTask(BasePipelineTask):
|
|||||||
await self._sink.setup(setup)
|
await self._sink.setup(setup)
|
||||||
|
|
||||||
async def _cleanup(self, cleanup_pipeline: bool):
|
async def _cleanup(self, cleanup_pipeline: bool):
|
||||||
|
"""Clean up the pipeline task and processors."""
|
||||||
# Cleanup base object.
|
# Cleanup base object.
|
||||||
await self.cleanup()
|
await self.cleanup()
|
||||||
|
|
||||||
@@ -559,10 +618,11 @@ class PipelineTask(BasePipelineTask):
|
|||||||
await self._sink.cleanup()
|
await self._sink.cleanup()
|
||||||
|
|
||||||
async def _process_push_queue(self):
|
async def _process_push_queue(self):
|
||||||
"""This is the task that runs the pipeline for the first time by sending
|
"""Process frames from the push queue and send them through the pipeline.
|
||||||
|
|
||||||
|
This is the task that runs the pipeline for the first time by sending
|
||||||
a StartFrame and by pushing any other frames queued by the user. It runs
|
a StartFrame and by pushing any other frames queued by the user. It runs
|
||||||
until the tasks is cancelled or stopped (e.g. with an EndFrame).
|
until the tasks is cancelled or stopped (e.g. with an EndFrame).
|
||||||
|
|
||||||
"""
|
"""
|
||||||
self._clock.start()
|
self._clock.start()
|
||||||
|
|
||||||
@@ -596,11 +656,12 @@ class PipelineTask(BasePipelineTask):
|
|||||||
await self._cleanup(cleanup_pipeline)
|
await self._cleanup(cleanup_pipeline)
|
||||||
|
|
||||||
async def _process_up_queue(self):
|
async def _process_up_queue(self):
|
||||||
"""This is the task that processes frames coming upstream from the
|
"""Process frames coming upstream from the pipeline.
|
||||||
|
|
||||||
|
This is the task that processes frames coming upstream from the
|
||||||
pipeline. These frames might indicate, for example, that we want the
|
pipeline. These frames might indicate, for example, that we want the
|
||||||
pipeline to be stopped (e.g. EndTaskFrame) in which case we would send
|
pipeline to be stopped (e.g. EndTaskFrame) in which case we would send
|
||||||
an EndFrame down the pipeline.
|
an EndFrame down the pipeline.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
frame = await self._up_queue.get()
|
frame = await self._up_queue.get()
|
||||||
@@ -629,11 +690,12 @@ class PipelineTask(BasePipelineTask):
|
|||||||
self._up_queue.task_done()
|
self._up_queue.task_done()
|
||||||
|
|
||||||
async def _process_down_queue(self):
|
async def _process_down_queue(self):
|
||||||
"""This tasks process frames coming downstream from the pipeline. For
|
"""Process frames coming downstream from the pipeline.
|
||||||
|
|
||||||
|
This tasks process frames coming downstream from the pipeline. For
|
||||||
example, heartbeat frames or an EndFrame which would indicate all
|
example, heartbeat frames or an EndFrame which would indicate all
|
||||||
processors have handled the EndFrame and therefore we can exit the task
|
processors have handled the EndFrame and therefore we can exit the task
|
||||||
cleanly.
|
cleanly.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
frame = await self._down_queue.get()
|
frame = await self._down_queue.get()
|
||||||
@@ -664,7 +726,7 @@ class PipelineTask(BasePipelineTask):
|
|||||||
self._down_queue.task_done()
|
self._down_queue.task_done()
|
||||||
|
|
||||||
async def _heartbeat_push_handler(self):
|
async def _heartbeat_push_handler(self):
|
||||||
"""This tasks pushes a heartbeat frame every heartbeat period."""
|
"""Push heartbeat frames at regular intervals."""
|
||||||
while True:
|
while True:
|
||||||
# Don't use `queue_frame()` because if an EndFrame is queued the
|
# Don't use `queue_frame()` because if an EndFrame is queued the
|
||||||
# task will just stop waiting for the pipeline to finish not
|
# task will just stop waiting for the pipeline to finish not
|
||||||
@@ -673,11 +735,12 @@ class PipelineTask(BasePipelineTask):
|
|||||||
await asyncio.sleep(self._params.heartbeats_period_secs)
|
await asyncio.sleep(self._params.heartbeats_period_secs)
|
||||||
|
|
||||||
async def _heartbeat_monitor_handler(self):
|
async def _heartbeat_monitor_handler(self):
|
||||||
"""This tasks monitors heartbeat frames. If a heartbeat frame has not
|
"""Monitor heartbeat frames for processing time and timeout detection.
|
||||||
|
|
||||||
|
This task monitors heartbeat frames. If a heartbeat frame has not
|
||||||
been received for a long period a warning will be logged. It also logs
|
been received for a long period a warning will be logged. It also logs
|
||||||
the time that a heartbeat frame takes to processes, that is how long it
|
the time that a heartbeat frame takes to processes, that is how long it
|
||||||
takes for the heartbeat frame to traverse all the pipeline.
|
takes for the heartbeat frame to traverse all the pipeline.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
wait_time = HEARTBEAT_MONITOR_SECONDS
|
wait_time = HEARTBEAT_MONITOR_SECONDS
|
||||||
while True:
|
while True:
|
||||||
@@ -692,9 +755,12 @@ class PipelineTask(BasePipelineTask):
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def _idle_monitor_handler(self):
|
async def _idle_monitor_handler(self):
|
||||||
"""This tasks monitors activity in the pipeline. If no frames are
|
"""Monitor pipeline activity and detect idle conditions.
|
||||||
received (heartbeats don't count) the pipeline is considered idle.
|
|
||||||
|
|
||||||
|
Tracks frame activity and triggers idle timeout events when the
|
||||||
|
pipeline hasn't received relevant frames within the timeout period.
|
||||||
|
|
||||||
|
Note: Heartbeats are excluded from idle detection.
|
||||||
"""
|
"""
|
||||||
running = True
|
running = True
|
||||||
last_frame_time = 0
|
last_frame_time = 0
|
||||||
@@ -732,10 +798,13 @@ class PipelineTask(BasePipelineTask):
|
|||||||
running = await self._idle_timeout_detected(frame_buffer)
|
running = await self._idle_timeout_detected(frame_buffer)
|
||||||
|
|
||||||
async def _idle_timeout_detected(self, last_frames: Deque[Frame]) -> bool:
|
async def _idle_timeout_detected(self, last_frames: Deque[Frame]) -> bool:
|
||||||
"""Logic for when the pipeline is idle.
|
"""Handle idle timeout detection and optional cancellation.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
last_frames: Recent frames received before timeout for debugging.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
bool: Whther the pipeline task is being cancelled or not.
|
Whether the pipeline task should continue running.
|
||||||
"""
|
"""
|
||||||
logger.warning("Idle timeout detected. Last 10 frames received:")
|
logger.warning("Idle timeout detected. Last 10 frames received:")
|
||||||
for i, frame in enumerate(last_frames, 1):
|
for i, frame in enumerate(last_frames, 1):
|
||||||
@@ -749,6 +818,7 @@ class PipelineTask(BasePipelineTask):
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
def _print_dangling_tasks(self):
|
def _print_dangling_tasks(self):
|
||||||
|
"""Log any dangling tasks that haven't been properly cleaned up."""
|
||||||
tasks = [t.get_name() for t in self._task_manager.current_tasks()]
|
tasks = [t.get_name() for t in self._task_manager.current_tasks()]
|
||||||
if tasks:
|
if tasks:
|
||||||
logger.warning(f"Dangling tasks detected: {tasks}")
|
logger.warning(f"Dangling tasks detected: {tasks}")
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Task observer for managing pipeline frame observers.
|
||||||
|
|
||||||
|
This module provides a proxy observer system that manages multiple observers
|
||||||
|
for pipeline frame events, ensuring that observer processing doesn't block
|
||||||
|
the main pipeline execution.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import inspect
|
import inspect
|
||||||
from typing import Dict, List, Optional
|
from typing import Dict, List, Optional
|
||||||
@@ -17,9 +24,15 @@ from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Proxy:
|
class Proxy:
|
||||||
"""This is the data we receive from the main observer and that we put into
|
"""Proxy data for managing observer tasks and queues.
|
||||||
a queue for later processing.
|
|
||||||
|
|
||||||
|
This represents is the data received from the main observer that
|
||||||
|
is queued for later processing.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
queue: Queue for frame data awaiting observer processing.
|
||||||
|
task: Asyncio task running the observer's frame processing loop.
|
||||||
|
observer: The actual observer instance being proxied.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
queue: asyncio.Queue
|
queue: asyncio.Queue
|
||||||
@@ -28,7 +41,9 @@ class Proxy:
|
|||||||
|
|
||||||
|
|
||||||
class TaskObserver(BaseObserver):
|
class TaskObserver(BaseObserver):
|
||||||
"""This is a pipeline frame observer that is meant to be used as a proxy to
|
"""Proxy observer that manages multiple observers without blocking the pipeline.
|
||||||
|
|
||||||
|
This is a pipeline frame observer that is meant to be used as a proxy to
|
||||||
the user provided observers. That is, this is the observer that should be
|
the user provided observers. That is, this is the observer that should be
|
||||||
passed to the frame processors. Then, every time a frame is pushed this
|
passed to the frame processors. Then, every time a frame is pushed this
|
||||||
observer will call all the observers registered to the pipeline task.
|
observer will call all the observers registered to the pipeline task.
|
||||||
@@ -37,7 +52,6 @@ class TaskObserver(BaseObserver):
|
|||||||
pipeline by creating a queue and a task for each user observer. When a frame
|
pipeline by creating a queue and a task for each user observer. When a frame
|
||||||
is received, it will be put in a queue for efficiency and later processed by
|
is received, it will be put in a queue for efficiency and later processed by
|
||||||
each task.
|
each task.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -47,6 +61,13 @@ class TaskObserver(BaseObserver):
|
|||||||
task_manager: BaseTaskManager,
|
task_manager: BaseTaskManager,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the TaskObserver.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
observers: List of observers to manage. Defaults to empty list.
|
||||||
|
task_manager: Task manager for creating and managing observer tasks.
|
||||||
|
**kwargs: Additional arguments passed to the base observer.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._observers = observers or []
|
self._observers = observers or []
|
||||||
self._task_manager = task_manager
|
self._task_manager = task_manager
|
||||||
@@ -55,6 +76,11 @@ class TaskObserver(BaseObserver):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def add_observer(self, observer: BaseObserver):
|
def add_observer(self, observer: BaseObserver):
|
||||||
|
"""Add a new observer to the managed list.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
observer: The observer to add.
|
||||||
|
"""
|
||||||
# Add the observer to the list.
|
# Add the observer to the list.
|
||||||
self._observers.append(observer)
|
self._observers.append(observer)
|
||||||
|
|
||||||
@@ -65,6 +91,11 @@ class TaskObserver(BaseObserver):
|
|||||||
self._proxies[observer] = proxy
|
self._proxies[observer] = proxy
|
||||||
|
|
||||||
async def remove_observer(self, observer: BaseObserver):
|
async def remove_observer(self, observer: BaseObserver):
|
||||||
|
"""Remove an observer and clean up its resources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
observer: The observer to remove.
|
||||||
|
"""
|
||||||
# If the observer has a proxy, remove it.
|
# If the observer has a proxy, remove it.
|
||||||
if observer in self._proxies:
|
if observer in self._proxies:
|
||||||
proxy = self._proxies[observer]
|
proxy = self._proxies[observer]
|
||||||
@@ -78,11 +109,11 @@ class TaskObserver(BaseObserver):
|
|||||||
self._observers.remove(observer)
|
self._observers.remove(observer)
|
||||||
|
|
||||||
async def start(self):
|
async def start(self):
|
||||||
"""Starts all proxy observer tasks."""
|
"""Start all proxy observer tasks."""
|
||||||
self._proxies = self._create_proxies(self._observers)
|
self._proxies = self._create_proxies(self._observers)
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
"""Stops all proxy observer tasks."""
|
"""Stop all proxy observer tasks."""
|
||||||
if not self._proxies:
|
if not self._proxies:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -90,13 +121,20 @@ class TaskObserver(BaseObserver):
|
|||||||
await self._task_manager.cancel_task(proxy.task)
|
await self._task_manager.cancel_task(proxy.task)
|
||||||
|
|
||||||
async def on_push_frame(self, data: FramePushed):
|
async def on_push_frame(self, data: FramePushed):
|
||||||
|
"""Queue frame data for all managed observers.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: The frame push event data to distribute to observers.
|
||||||
|
"""
|
||||||
for proxy in self._proxies.values():
|
for proxy in self._proxies.values():
|
||||||
await proxy.queue.put(data)
|
await proxy.queue.put(data)
|
||||||
|
|
||||||
def _started(self) -> bool:
|
def _started(self) -> bool:
|
||||||
|
"""Check if the task observer has been started."""
|
||||||
return self._proxies is not None
|
return self._proxies is not None
|
||||||
|
|
||||||
def _create_proxy(self, observer: BaseObserver) -> Proxy:
|
def _create_proxy(self, observer: BaseObserver) -> Proxy:
|
||||||
|
"""Create a proxy for a single observer."""
|
||||||
queue = WatchdogQueue(self._task_manager)
|
queue = WatchdogQueue(self._task_manager)
|
||||||
task = self._task_manager.create_task(
|
task = self._task_manager.create_task(
|
||||||
self._proxy_task_handler(queue, observer),
|
self._proxy_task_handler(queue, observer),
|
||||||
@@ -106,6 +144,7 @@ class TaskObserver(BaseObserver):
|
|||||||
return proxy
|
return proxy
|
||||||
|
|
||||||
def _create_proxies(self, observers: List[BaseObserver]) -> Dict[BaseObserver, Proxy]:
|
def _create_proxies(self, observers: List[BaseObserver]) -> Dict[BaseObserver, Proxy]:
|
||||||
|
"""Create proxies for all observers."""
|
||||||
proxies = {}
|
proxies = {}
|
||||||
for observer in observers:
|
for observer in observers:
|
||||||
proxy = self._create_proxy(observer)
|
proxy = self._create_proxy(observer)
|
||||||
@@ -113,6 +152,7 @@ class TaskObserver(BaseObserver):
|
|||||||
return proxies
|
return proxies
|
||||||
|
|
||||||
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):
|
async def _proxy_task_handler(self, queue: asyncio.Queue, observer: BaseObserver):
|
||||||
|
"""Handle frame processing for a single observer."""
|
||||||
warning_reported = False
|
warning_reported = False
|
||||||
while True:
|
while True:
|
||||||
data = await queue.get()
|
data = await queue.get()
|
||||||
|
|||||||
@@ -1,3 +1,16 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Sequential pipeline merging for Pipecat.
|
||||||
|
|
||||||
|
This module provides a pipeline implementation that sequentially merges
|
||||||
|
the output from multiple pipelines, processing them one after another
|
||||||
|
in a specified order.
|
||||||
|
"""
|
||||||
|
|
||||||
from typing import List
|
from typing import List
|
||||||
|
|
||||||
from pipecat.frames.frames import EndFrame, EndPipeFrame
|
from pipecat.frames.frames import EndFrame, EndPipeFrame
|
||||||
@@ -5,14 +18,31 @@ from pipecat.pipeline.pipeline import Pipeline
|
|||||||
|
|
||||||
|
|
||||||
class SequentialMergePipeline(Pipeline):
|
class SequentialMergePipeline(Pipeline):
|
||||||
"""This class merges the sink queues from a list of pipelines. Frames from
|
"""Pipeline that sequentially merges output from multiple pipelines.
|
||||||
each pipeline's sink are merged in the order of pipelines in the list."""
|
|
||||||
|
This pipeline merges the sink queues from a list of pipelines by processing
|
||||||
|
frames from each pipeline's sink sequentially in the order specified. Each
|
||||||
|
pipeline runs to completion before the next one begins processing.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, pipelines: List[Pipeline]):
|
def __init__(self, pipelines: List[Pipeline]):
|
||||||
|
"""Initialize the sequential merge pipeline.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
pipelines: List of pipelines to merge sequentially. Pipelines will
|
||||||
|
be processed in the order they appear in this list.
|
||||||
|
"""
|
||||||
super().__init__([])
|
super().__init__([])
|
||||||
self.pipelines = pipelines
|
self.pipelines = pipelines
|
||||||
|
|
||||||
async def run_pipeline(self):
|
async def run_pipeline(self):
|
||||||
|
"""Run all pipelines sequentially and merge their output.
|
||||||
|
|
||||||
|
Processes each pipeline in order, consuming all frames from each
|
||||||
|
pipeline's sink until an EndFrame or EndPipeFrame is encountered,
|
||||||
|
then moves to the next pipeline. After all pipelines complete,
|
||||||
|
sends a final EndFrame to signal completion.
|
||||||
|
"""
|
||||||
for idx, pipeline in enumerate(self.pipelines):
|
for idx, pipeline in enumerate(self.pipelines):
|
||||||
while True:
|
while True:
|
||||||
frame = await pipeline.sink.get()
|
frame = await pipeline.sink.get()
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ class DTMFAggregator(FrameProcessor):
|
|||||||
|
|
||||||
The aggregator accumulates digits from InputDTMFFrame instances and flushes
|
The aggregator accumulates digits from InputDTMFFrame instances and flushes
|
||||||
when:
|
when:
|
||||||
|
|
||||||
- Timeout occurs (configurable idle period)
|
- Timeout occurs (configurable idle period)
|
||||||
- Termination digit is received (default: '#')
|
- Termination digit is received (default: '#')
|
||||||
- EndFrame or CancelFrame is received
|
- EndFrame or CancelFrame is received
|
||||||
|
|||||||
@@ -92,7 +92,7 @@ class LLMFullResponseAggregator(FrameProcessor):
|
|||||||
the complete response via an event handler.
|
the complete response via an event handler.
|
||||||
|
|
||||||
The aggregator provides an "on_completion" event that fires when a full
|
The aggregator provides an "on_completion" event that fires when a full
|
||||||
completion is available:
|
completion is available::
|
||||||
|
|
||||||
@aggregator.event_handler("on_completion")
|
@aggregator.event_handler("on_completion")
|
||||||
async def on_completion(
|
async def on_completion(
|
||||||
@@ -363,6 +363,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator):
|
|||||||
|
|
||||||
This aggregator handles the complex logic of aggregating user speech transcriptions
|
This aggregator handles the complex logic of aggregating user speech transcriptions
|
||||||
from STT services. It manages multiple scenarios including:
|
from STT services. It manages multiple scenarios including:
|
||||||
|
|
||||||
- Transcriptions received between VAD events
|
- Transcriptions received between VAD events
|
||||||
- Transcriptions received outside VAD events
|
- Transcriptions received outside VAD events
|
||||||
- Interim vs final transcriptions
|
- Interim vs final transcriptions
|
||||||
@@ -654,6 +655,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator):
|
|||||||
"""Assistant LLM aggregator that processes bot responses and function calls.
|
"""Assistant LLM aggregator that processes bot responses and function calls.
|
||||||
|
|
||||||
This aggregator handles the complex logic of processing assistant responses including:
|
This aggregator handles the complex logic of processing assistant responses including:
|
||||||
|
|
||||||
- Text frame aggregation between response start/end markers
|
- Text frame aggregation between response start/end markers
|
||||||
- Function call lifecycle management
|
- Function call lifecycle management
|
||||||
- Context updates with timestamps
|
- Context updates with timestamps
|
||||||
|
|||||||
@@ -210,9 +210,10 @@ class OpenAILLMContext:
|
|||||||
def from_standard_message(self, message):
|
def from_standard_message(self, message):
|
||||||
"""Convert from OpenAI message format to OpenAI message format (passthrough).
|
"""Convert from OpenAI message format to OpenAI message format (passthrough).
|
||||||
|
|
||||||
OpenAI's format allows both simple string content and structured content:
|
OpenAI's format allows both simple string content and structured content::
|
||||||
- Simple: {"role": "user", "content": "Hello"}
|
|
||||||
- Structured: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
|
Simple: {"role": "user", "content": "Hello"}
|
||||||
|
Structured: {"role": "user", "content": [{"type": "text", "text": "Hello"}]}
|
||||||
|
|
||||||
Since OpenAI is our standard format, this is a passthrough function.
|
Since OpenAI is our standard format, this is a passthrough function.
|
||||||
|
|
||||||
|
|||||||
@@ -39,17 +39,19 @@ class AudioBufferProcessor(FrameProcessor):
|
|||||||
including sample rate conversion and mono/stereo output.
|
including sample rate conversion and mono/stereo output.
|
||||||
|
|
||||||
Events:
|
Events:
|
||||||
on_audio_data: Triggered when buffer_size is reached, providing merged audio
|
|
||||||
on_track_audio_data: Triggered when buffer_size is reached, providing separate tracks
|
- on_audio_data: Triggered when buffer_size is reached, providing merged audio
|
||||||
on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio
|
- on_track_audio_data: Triggered when buffer_size is reached, providing separate tracks
|
||||||
on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio
|
- on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio
|
||||||
|
- on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio
|
||||||
|
|
||||||
Audio handling:
|
Audio handling:
|
||||||
- Mono output (num_channels=1): User and bot audio are mixed
|
|
||||||
- Stereo output (num_channels=2): User audio on left, bot audio on right
|
- Mono output (num_channels=1): User and bot audio are mixed
|
||||||
- Automatic resampling of incoming audio to match desired sample_rate
|
- Stereo output (num_channels=2): User audio on left, bot audio on right
|
||||||
- Silence insertion for non-continuous audio streams
|
- Automatic resampling of incoming audio to match desired sample_rate
|
||||||
- Buffer synchronization between user and bot audio
|
- Silence insertion for non-continuous audio streams
|
||||||
|
- Buffer synchronization between user and bot audio
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
|
|||||||
@@ -84,6 +84,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
|
|||||||
|
|
||||||
This processor aggregates TTS text frames into complete utterances and emits them as
|
This processor aggregates TTS text frames into complete utterances and emits them as
|
||||||
transcript messages. Utterances are completed when:
|
transcript messages. Utterances are completed when:
|
||||||
|
|
||||||
- The bot stops speaking (BotStoppedSpeakingFrame)
|
- The bot stops speaking (BotStoppedSpeakingFrame)
|
||||||
- The bot is interrupted (StartInterruptionFrame)
|
- The bot is interrupted (StartInterruptionFrame)
|
||||||
- The pipeline ends (EndFrame)
|
- The pipeline ends (EndFrame)
|
||||||
@@ -108,34 +109,34 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
|
|||||||
TTS services with different formatting patterns.
|
TTS services with different formatting patterns.
|
||||||
|
|
||||||
Examples:
|
Examples:
|
||||||
Fragments with embedded spacing (concatenated):
|
Fragments with embedded spacing (concatenated)::
|
||||||
```
|
|
||||||
TTSTextFrame: ["Hello"]
|
TTSTextFrame: ["Hello"]
|
||||||
TTSTextFrame: [" there"] # Leading space
|
TTSTextFrame: [" there"] # Leading space
|
||||||
TTSTextFrame: ["!"]
|
TTSTextFrame: ["!"]
|
||||||
TTSTextFrame: [" How"] # Leading space
|
TTSTextFrame: [" How"] # Leading space
|
||||||
TTSTextFrame: ["'s"]
|
TTSTextFrame: ["'s"]
|
||||||
TTSTextFrame: [" it"] # Leading space
|
TTSTextFrame: [" it"] # Leading space
|
||||||
```
|
|
||||||
Result: "Hello there! How's it"
|
Result: "Hello there! How's it"
|
||||||
|
|
||||||
Fragments with trailing spaces (concatenated):
|
Fragments with trailing spaces (concatenated)::
|
||||||
```
|
|
||||||
TTSTextFrame: ["Hel"]
|
TTSTextFrame: ["Hel"]
|
||||||
TTSTextFrame: ["lo "] # Trailing space
|
TTSTextFrame: ["lo "] # Trailing space
|
||||||
TTSTextFrame: ["to "] # Trailing space
|
TTSTextFrame: ["to "] # Trailing space
|
||||||
TTSTextFrame: ["you"]
|
TTSTextFrame: ["you"]
|
||||||
```
|
|
||||||
Result: "Hello to you"
|
Result: "Hello to you"
|
||||||
|
|
||||||
Word-by-word fragments without spacing (joined with spaces):
|
Word-by-word fragments without spacing (joined with spaces)::
|
||||||
```
|
|
||||||
TTSTextFrame: ["Hello"]
|
TTSTextFrame: ["Hello"]
|
||||||
TTSTextFrame: ["there"]
|
TTSTextFrame: ["there"]
|
||||||
TTSTextFrame: ["how"]
|
TTSTextFrame: ["how"]
|
||||||
TTSTextFrame: ["are"]
|
TTSTextFrame: ["are"]
|
||||||
TTSTextFrame: ["you"]
|
TTSTextFrame: ["you"]
|
||||||
```
|
|
||||||
Result: "Hello there how are you"
|
Result: "Hello there how are you"
|
||||||
"""
|
"""
|
||||||
if self._current_text_parts and self._aggregation_start_time:
|
if self._current_text_parts and self._aggregation_start_time:
|
||||||
@@ -179,6 +180,7 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor):
|
|||||||
"""Process frames into assistant conversation messages.
|
"""Process frames into assistant conversation messages.
|
||||||
|
|
||||||
Handles different frame types:
|
Handles different frame types:
|
||||||
|
|
||||||
- TTSTextFrame: Aggregates text for current utterance
|
- TTSTextFrame: Aggregates text for current utterance
|
||||||
- BotStoppedSpeakingFrame: Completes current utterance
|
- BotStoppedSpeakingFrame: Completes current utterance
|
||||||
- StartInterruptionFrame: Completes current utterance due to interruption
|
- StartInterruptionFrame: Completes current utterance due to interruption
|
||||||
@@ -221,8 +223,8 @@ class TranscriptProcessor:
|
|||||||
Provides unified access to user and assistant transcript processors
|
Provides unified access to user and assistant transcript processors
|
||||||
with shared event handling.
|
with shared event handling.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
```python
|
|
||||||
transcript = TranscriptProcessor()
|
transcript = TranscriptProcessor()
|
||||||
|
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(
|
||||||
@@ -242,7 +244,6 @@ class TranscriptProcessor:
|
|||||||
@transcript.event_handler("on_transcript_update")
|
@transcript.event_handler("on_transcript_update")
|
||||||
async def handle_update(processor, frame):
|
async def handle_update(processor, frame):
|
||||||
print(f"New messages: {frame.messages}")
|
print(f"New messages: {frame.messages}")
|
||||||
```
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
|||||||
@@ -28,8 +28,8 @@ class UserIdleProcessor(FrameProcessor):
|
|||||||
users become idle. It starts monitoring only after the first conversation
|
users become idle. It starts monitoring only after the first conversation
|
||||||
activity and supports both basic and retry-based callback patterns.
|
activity and supports both basic and retry-based callback patterns.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
```
|
|
||||||
# Retry callback:
|
# Retry callback:
|
||||||
async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool:
|
async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool:
|
||||||
if retry_count < 3:
|
if retry_count < 3:
|
||||||
@@ -45,7 +45,6 @@ class UserIdleProcessor(FrameProcessor):
|
|||||||
callback=handle_idle,
|
callback=handle_idle,
|
||||||
timeout=5.0
|
timeout=5.0
|
||||||
)
|
)
|
||||||
```
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -61,11 +60,10 @@ class UserIdleProcessor(FrameProcessor):
|
|||||||
"""Initialize the user idle processor.
|
"""Initialize the user idle processor.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
callback: Function to call when user is idle. Can be either:
|
callback: Function to call when user is idle. Can be either a basic
|
||||||
- Basic callback(processor) -> None
|
callback taking only the processor, or a retry callback taking
|
||||||
- Retry callback(processor, retry_count) -> bool
|
the processor and retry count. Retry callbacks should return
|
||||||
Return True to continue monitoring for idle events,
|
True to continue monitoring or False to stop.
|
||||||
Return False to stop the idle monitoring task
|
|
||||||
timeout: Seconds to wait before considering user idle.
|
timeout: Seconds to wait before considering user idle.
|
||||||
**kwargs: Additional arguments passed to FrameProcessor.
|
**kwargs: Additional arguments passed to FrameProcessor.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Frame serialization interfaces for Pipecat."""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
|
|
||||||
@@ -11,23 +13,63 @@ from pipecat.frames.frames import Frame, StartFrame
|
|||||||
|
|
||||||
|
|
||||||
class FrameSerializerType(Enum):
|
class FrameSerializerType(Enum):
|
||||||
|
"""Enumeration of supported frame serialization formats.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
BINARY: Binary serialization format for compact representation.
|
||||||
|
TEXT: Text-based serialization format for human-readable output.
|
||||||
|
"""
|
||||||
|
|
||||||
BINARY = "binary"
|
BINARY = "binary"
|
||||||
TEXT = "text"
|
TEXT = "text"
|
||||||
|
|
||||||
|
|
||||||
class FrameSerializer(ABC):
|
class FrameSerializer(ABC):
|
||||||
|
"""Abstract base class for frame serialization implementations.
|
||||||
|
|
||||||
|
Defines the interface for converting frames to/from serialized formats
|
||||||
|
for transmission or storage. Subclasses must implement serialization
|
||||||
|
type detection and the core serialize/deserialize methods.
|
||||||
|
"""
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def type(self) -> FrameSerializerType:
|
def type(self) -> FrameSerializerType:
|
||||||
|
"""Get the serialization type supported by this serializer.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The FrameSerializerType indicating binary or text format.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def setup(self, frame: StartFrame):
|
async def setup(self, frame: StartFrame):
|
||||||
|
"""Initialize the serializer with startup configuration.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: StartFrame containing initialization parameters.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def serialize(self, frame: Frame) -> str | bytes | None:
|
async def serialize(self, frame: Frame) -> str | bytes | None:
|
||||||
|
"""Convert a frame to its serialized representation.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to serialize.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Serialized frame data as string, bytes, or None if serialization fails.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def deserialize(self, data: str | bytes) -> Frame | None:
|
async def deserialize(self, data: str | bytes) -> Frame | None:
|
||||||
|
"""Convert serialized data back to a frame object.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: Serialized frame data as string or bytes.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Reconstructed Frame object, or None if deserialization fails.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Exotel Media Streams serializer for Pipecat."""
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -33,13 +35,14 @@ class ExotelFrameSerializer(FrameSerializer):
|
|||||||
media streams protocol. It supports audio conversion, DTMF events, and automatic
|
media streams protocol. It supports audio conversion, DTMF events, and automatic
|
||||||
call termination.
|
call termination.
|
||||||
|
|
||||||
Ref Doc for events - https://support.exotel.com/support/solutions/articles/3000108630-working-with-the-stream-and-voicebot-applet
|
Note: Ref docs for events:
|
||||||
|
https://support.exotel.com/support/solutions/articles/3000108630-working-with-the-stream-and-voicebot-applet
|
||||||
"""
|
"""
|
||||||
|
|
||||||
class InputParams(BaseModel):
|
class InputParams(BaseModel):
|
||||||
"""Configuration parameters for ExotelFrameSerializer.
|
"""Configuration parameters for ExotelFrameSerializer.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
exotel_sample_rate: Sample rate used by Exotel, defaults to 8000 Hz.
|
exotel_sample_rate: Sample rate used by Exotel, defaults to 8000 Hz.
|
||||||
sample_rate: Optional override for pipeline input sample rate.
|
sample_rate: Optional override for pipeline input sample rate.
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""LiveKit frame serializer for Pipecat."""
|
||||||
|
|
||||||
import ctypes
|
import ctypes
|
||||||
import pickle
|
import pickle
|
||||||
|
|
||||||
@@ -21,11 +23,33 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class LivekitFrameSerializer(FrameSerializer):
|
class LivekitFrameSerializer(FrameSerializer):
|
||||||
|
"""Serializer for converting between Pipecat frames and LiveKit audio frames.
|
||||||
|
|
||||||
|
This serializer handles the conversion of Pipecat's OutputAudioRawFrame objects
|
||||||
|
to LiveKit AudioFrame objects for transmission, and the reverse conversion
|
||||||
|
for received audio data.
|
||||||
|
"""
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def type(self) -> FrameSerializerType:
|
def type(self) -> FrameSerializerType:
|
||||||
|
"""Get the serializer type.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The serializer type indicating binary serialization.
|
||||||
|
"""
|
||||||
return FrameSerializerType.BINARY
|
return FrameSerializerType.BINARY
|
||||||
|
|
||||||
async def serialize(self, frame: Frame) -> str | bytes | None:
|
async def serialize(self, frame: Frame) -> str | bytes | None:
|
||||||
|
"""Serialize a Pipecat frame to LiveKit AudioFrame format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The Pipecat frame to serialize. Only OutputAudioRawFrame
|
||||||
|
instances are supported.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Pickled LiveKit AudioFrame bytes if frame is OutputAudioRawFrame,
|
||||||
|
None otherwise.
|
||||||
|
"""
|
||||||
if not isinstance(frame, OutputAudioRawFrame):
|
if not isinstance(frame, OutputAudioRawFrame):
|
||||||
return None
|
return None
|
||||||
audio_frame = AudioFrame(
|
audio_frame = AudioFrame(
|
||||||
@@ -37,6 +61,15 @@ class LivekitFrameSerializer(FrameSerializer):
|
|||||||
return pickle.dumps(audio_frame)
|
return pickle.dumps(audio_frame)
|
||||||
|
|
||||||
async def deserialize(self, data: str | bytes) -> Frame | None:
|
async def deserialize(self, data: str | bytes) -> Frame | None:
|
||||||
|
"""Deserialize LiveKit AudioFrame data to a Pipecat frame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: Pickled data containing a LiveKit AudioFrame.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
InputAudioRawFrame containing the deserialized audio data,
|
||||||
|
or None if deserialization fails.
|
||||||
|
"""
|
||||||
audio_frame: AudioFrame = pickle.loads(data)["frame"]
|
audio_frame: AudioFrame = pickle.loads(data)["frame"]
|
||||||
return InputAudioRawFrame(
|
return InputAudioRawFrame(
|
||||||
audio=bytes(audio_frame.data),
|
audio=bytes(audio_frame.data),
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Plivo WebSocket frame serializer for audio streaming."""
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -38,22 +40,12 @@ class PlivoFrameSerializer(FrameSerializer):
|
|||||||
When auto_hang_up is enabled (default), the serializer will automatically terminate
|
When auto_hang_up is enabled (default), the serializer will automatically terminate
|
||||||
the Plivo call when an EndFrame or CancelFrame is processed, but requires Plivo
|
the Plivo call when an EndFrame or CancelFrame is processed, but requires Plivo
|
||||||
credentials to be provided.
|
credentials to be provided.
|
||||||
|
|
||||||
Attributes:
|
|
||||||
_stream_id: The Plivo Stream ID.
|
|
||||||
_call_id: The associated Plivo Call ID.
|
|
||||||
_auth_id: Plivo auth ID for API access.
|
|
||||||
_auth_token: Plivo authentication token for API access.
|
|
||||||
_params: Configuration parameters.
|
|
||||||
_plivo_sample_rate: Sample rate used by Plivo (typically 8kHz).
|
|
||||||
_sample_rate: Input sample rate for the pipeline.
|
|
||||||
_resampler: Audio resampler for format conversion.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
class InputParams(BaseModel):
|
class InputParams(BaseModel):
|
||||||
"""Configuration parameters for PlivoFrameSerializer.
|
"""Configuration parameters for PlivoFrameSerializer.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
plivo_sample_rate: Sample rate used by Plivo, defaults to 8000 Hz.
|
plivo_sample_rate: Sample rate used by Plivo, defaults to 8000 Hz.
|
||||||
sample_rate: Optional override for pipeline input sample rate.
|
sample_rate: Optional override for pipeline input sample rate.
|
||||||
auto_hang_up: Whether to automatically terminate call on EndFrame.
|
auto_hang_up: Whether to automatically terminate call on EndFrame.
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Protobuf frame serialization for Pipecat."""
|
||||||
|
|
||||||
import dataclasses
|
import dataclasses
|
||||||
import json
|
import json
|
||||||
|
|
||||||
@@ -22,13 +24,25 @@ from pipecat.frames.frames import (
|
|||||||
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
|
from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType
|
||||||
|
|
||||||
|
|
||||||
# Data class for converting transport messages into Protobuf format.
|
|
||||||
@dataclasses.dataclass
|
@dataclasses.dataclass
|
||||||
class MessageFrame:
|
class MessageFrame:
|
||||||
|
"""Data class for converting transport messages into Protobuf format.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
data: JSON-encoded message data for transport.
|
||||||
|
"""
|
||||||
|
|
||||||
data: str
|
data: str
|
||||||
|
|
||||||
|
|
||||||
class ProtobufFrameSerializer(FrameSerializer):
|
class ProtobufFrameSerializer(FrameSerializer):
|
||||||
|
"""Serializer for converting Pipecat frames to/from Protocol Buffer format.
|
||||||
|
|
||||||
|
Provides efficient binary serialization for frame transport over network
|
||||||
|
connections. Supports text, audio, transcription, and message frames with
|
||||||
|
automatic conversion between transport message types.
|
||||||
|
"""
|
||||||
|
|
||||||
SERIALIZABLE_TYPES = {
|
SERIALIZABLE_TYPES = {
|
||||||
TextFrame: "text",
|
TextFrame: "text",
|
||||||
OutputAudioRawFrame: "audio",
|
OutputAudioRawFrame: "audio",
|
||||||
@@ -46,13 +60,27 @@ class ProtobufFrameSerializer(FrameSerializer):
|
|||||||
DESERIALIZABLE_FIELDS = {v: k for k, v in DESERIALIZABLE_TYPES.items()}
|
DESERIALIZABLE_FIELDS = {v: k for k, v in DESERIALIZABLE_TYPES.items()}
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
"""Initialize the Protobuf frame serializer."""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def type(self) -> FrameSerializerType:
|
def type(self) -> FrameSerializerType:
|
||||||
|
"""Get the serializer type.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
FrameSerializerType.BINARY indicating binary serialization format.
|
||||||
|
"""
|
||||||
return FrameSerializerType.BINARY
|
return FrameSerializerType.BINARY
|
||||||
|
|
||||||
async def serialize(self, frame: Frame) -> str | bytes | None:
|
async def serialize(self, frame: Frame) -> str | bytes | None:
|
||||||
|
"""Serialize a frame to Protocol Buffer binary format.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to serialize.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Serialized frame as bytes, or None if frame type is not serializable.
|
||||||
|
"""
|
||||||
# Wrapping this messages as a JSONFrame to send
|
# Wrapping this messages as a JSONFrame to send
|
||||||
if isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
|
if isinstance(frame, (TransportMessageFrame, TransportMessageUrgentFrame)):
|
||||||
frame = MessageFrame(
|
frame = MessageFrame(
|
||||||
@@ -75,6 +103,14 @@ class ProtobufFrameSerializer(FrameSerializer):
|
|||||||
return proto_frame.SerializeToString()
|
return proto_frame.SerializeToString()
|
||||||
|
|
||||||
async def deserialize(self, data: str | bytes) -> Frame | None:
|
async def deserialize(self, data: str | bytes) -> Frame | None:
|
||||||
|
"""Deserialize Protocol Buffer binary data to a frame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: Binary protobuf data to deserialize.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Deserialized frame instance, or None if deserialization fails.
|
||||||
|
"""
|
||||||
proto = frame_protos.Frame.FromString(data)
|
proto = frame_protos.Frame.FromString(data)
|
||||||
which = proto.WhichOneof("frame")
|
which = proto.WhichOneof("frame")
|
||||||
if which not in self.DESERIALIZABLE_FIELDS:
|
if which not in self.DESERIALIZABLE_FIELDS:
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Telnyx WebSocket frame serializer for Pipecat."""
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -43,22 +45,12 @@ class TelnyxFrameSerializer(FrameSerializer):
|
|||||||
When auto_hang_up is enabled (default), the serializer will automatically terminate
|
When auto_hang_up is enabled (default), the serializer will automatically terminate
|
||||||
the Telnyx call when an EndFrame or CancelFrame is processed, but requires Telnyx
|
the Telnyx call when an EndFrame or CancelFrame is processed, but requires Telnyx
|
||||||
credentials to be provided.
|
credentials to be provided.
|
||||||
|
|
||||||
Attributes:
|
|
||||||
_stream_id: The Telnyx Stream ID.
|
|
||||||
_call_control_id: The associated Telnyx Call Control ID.
|
|
||||||
_api_key: Telnyx API key for API access.
|
|
||||||
_params: Configuration parameters.
|
|
||||||
_telnyx_sample_rate: Sample rate used by Telnyx (typically 8kHz).
|
|
||||||
_sample_rate: Input sample rate for the pipeline.
|
|
||||||
_resampler: Audio resampler for format conversion.
|
|
||||||
_hangup_attempted: Flag to track if hang-up has been attempted.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
class InputParams(BaseModel):
|
class InputParams(BaseModel):
|
||||||
"""Configuration parameters for TelnyxFrameSerializer.
|
"""Configuration parameters for TelnyxFrameSerializer.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
telnyx_sample_rate: Sample rate used by Telnyx, defaults to 8000 Hz.
|
telnyx_sample_rate: Sample rate used by Telnyx, defaults to 8000 Hz.
|
||||||
sample_rate: Optional override for pipeline input sample rate.
|
sample_rate: Optional override for pipeline input sample rate.
|
||||||
inbound_encoding: Audio encoding for data sent to Telnyx (e.g., "PCMU").
|
inbound_encoding: Audio encoding for data sent to Telnyx (e.g., "PCMU").
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Twilio Media Streams WebSocket protocol serializer for Pipecat."""
|
||||||
|
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -38,22 +40,12 @@ class TwilioFrameSerializer(FrameSerializer):
|
|||||||
When auto_hang_up is enabled (default), the serializer will automatically terminate
|
When auto_hang_up is enabled (default), the serializer will automatically terminate
|
||||||
the Twilio call when an EndFrame or CancelFrame is processed, but requires Twilio
|
the Twilio call when an EndFrame or CancelFrame is processed, but requires Twilio
|
||||||
credentials to be provided.
|
credentials to be provided.
|
||||||
|
|
||||||
Attributes:
|
|
||||||
_stream_sid: The Twilio Media Stream SID.
|
|
||||||
_call_sid: The associated Twilio Call SID.
|
|
||||||
_account_sid: Twilio account SID for API access.
|
|
||||||
_auth_token: Twilio authentication token for API access.
|
|
||||||
_params: Configuration parameters.
|
|
||||||
_twilio_sample_rate: Sample rate used by Twilio (typically 8kHz).
|
|
||||||
_sample_rate: Input sample rate for the pipeline.
|
|
||||||
_resampler: Audio resampler for format conversion.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
class InputParams(BaseModel):
|
class InputParams(BaseModel):
|
||||||
"""Configuration parameters for TwilioFrameSerializer.
|
"""Configuration parameters for TwilioFrameSerializer.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
twilio_sample_rate: Sample rate used by Twilio, defaults to 8000 Hz.
|
twilio_sample_rate: Sample rate used by Twilio, defaults to 8000 Hz.
|
||||||
sample_rate: Optional override for pipeline input sample rate.
|
sample_rate: Optional override for pipeline input sample rate.
|
||||||
auto_hang_up: Whether to automatically terminate call on EndFrame.
|
auto_hang_up: Whether to automatically terminate call on EndFrame.
|
||||||
|
|||||||
@@ -538,20 +538,37 @@ class AnthropicLLMContext(OpenAILLMContext):
|
|||||||
Handles text content and function calls for both user and assistant messages.
|
Handles text content and function calls for both user and assistant messages.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
obj: Message in Anthropic format:
|
obj: Message in Anthropic format.
|
||||||
{
|
|
||||||
"role": "user/assistant",
|
|
||||||
"content": str | [{"type": "text/tool_use/tool_result", ...}]
|
|
||||||
}
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of messages in standard format:
|
List of messages in standard format.
|
||||||
[
|
|
||||||
|
Examples:
|
||||||
|
Input Anthropic format::
|
||||||
|
|
||||||
{
|
{
|
||||||
"role": "user/assistant/tool",
|
"role": "assistant",
|
||||||
"content": [{"type": "text", "text": str}]
|
"content": [
|
||||||
|
{"type": "text", "text": "Hello"},
|
||||||
|
{"type": "tool_use", "id": "123", "name": "search", "input": {"q": "test"}}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
|
||||||
|
Output standard format::
|
||||||
|
|
||||||
|
[
|
||||||
|
{"role": "assistant", "content": [{"type": "text", "text": "Hello"}]},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"tool_calls": [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"id": "123",
|
||||||
|
"function": {"name": "search", "arguments": '{"q": "test"}'}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
"""
|
"""
|
||||||
# todo: image format (?)
|
# todo: image format (?)
|
||||||
# tool_use
|
# tool_use
|
||||||
@@ -613,23 +630,37 @@ class AnthropicLLMContext(OpenAILLMContext):
|
|||||||
Empty text content is converted to "(empty)".
|
Empty text content is converted to "(empty)".
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
message: Message in standard format:
|
message: Message in standard format.
|
||||||
{
|
|
||||||
"role": "user/assistant/tool",
|
|
||||||
"content": str | [{"type": "text", ...}],
|
|
||||||
"tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}]
|
|
||||||
}
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Message in Anthropic format:
|
Message in Anthropic format.
|
||||||
{
|
|
||||||
"role": "user/assistant",
|
Examples:
|
||||||
"content": str | [
|
Input standard format::
|
||||||
{"type": "text", "text": str} |
|
|
||||||
{"type": "tool_use", "id": str, "name": str, "input": dict} |
|
{
|
||||||
{"type": "tool_result", "tool_use_id": str, "content": str}
|
"role": "assistant",
|
||||||
]
|
"tool_calls": [
|
||||||
}
|
{
|
||||||
|
"id": "123",
|
||||||
|
"function": {"name": "search", "arguments": '{"q": "test"}'}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
Output Anthropic format::
|
||||||
|
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "tool_use",
|
||||||
|
"id": "123",
|
||||||
|
"name": "search",
|
||||||
|
"input": {"q": "test"}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
"""
|
"""
|
||||||
# todo: image messages (?)
|
# todo: image messages (?)
|
||||||
if message["role"] == "tool":
|
if message["role"] == "tool":
|
||||||
|
|||||||
@@ -207,20 +207,37 @@ class AWSBedrockLLMContext(OpenAILLMContext):
|
|||||||
Handles text content and function calls for both user and assistant messages.
|
Handles text content and function calls for both user and assistant messages.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
obj: Message in AWS Bedrock format:
|
obj: Message in AWS Bedrock format.
|
||||||
{
|
|
||||||
"role": "user/assistant",
|
|
||||||
"content": [{"text": str} | {"toolUse": {...}} | {"toolResult": {...}}]
|
|
||||||
}
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of messages in standard format:
|
List of messages in standard format.
|
||||||
[
|
|
||||||
|
Examples:
|
||||||
|
AWS Bedrock format input::
|
||||||
|
|
||||||
{
|
{
|
||||||
"role": "user/assistant/tool",
|
"role": "assistant",
|
||||||
"content": [{"type": "text", "text": str}]
|
"content": [
|
||||||
|
{"text": "Hello"},
|
||||||
|
{"toolUse": {"toolUseId": "123", "name": "search", "input": {"q": "test"}}}
|
||||||
|
]
|
||||||
}
|
}
|
||||||
]
|
|
||||||
|
Standard format output::
|
||||||
|
|
||||||
|
[
|
||||||
|
{"role": "assistant", "content": [{"type": "text", "text": "Hello"}]},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"tool_calls": [
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"id": "123",
|
||||||
|
"function": {"name": "search", "arguments": '{"q": "test"}'}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
"""
|
"""
|
||||||
role = obj.get("role")
|
role = obj.get("role")
|
||||||
content = obj.get("content")
|
content = obj.get("content")
|
||||||
@@ -294,23 +311,38 @@ class AWSBedrockLLMContext(OpenAILLMContext):
|
|||||||
Empty text content is converted to "(empty)".
|
Empty text content is converted to "(empty)".
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
message: Message in standard format:
|
message: Message in standard format.
|
||||||
{
|
|
||||||
"role": "user/assistant/tool",
|
|
||||||
"content": str | [{"type": "text", ...}],
|
|
||||||
"tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}]
|
|
||||||
}
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Message in AWS Bedrock format:
|
Message in AWS Bedrock format.
|
||||||
{
|
|
||||||
"role": "user/assistant",
|
Examples:
|
||||||
"content": [
|
Standard format input::
|
||||||
{"text": str} |
|
|
||||||
{"toolUse": {"toolUseId": str, "name": str, "input": dict}} |
|
{
|
||||||
{"toolResult": {"toolUseId": str, "content": [...], "status": str}}
|
"role": "assistant",
|
||||||
]
|
"tool_calls": [
|
||||||
}
|
{
|
||||||
|
"id": "123",
|
||||||
|
"function": {"name": "search", "arguments": '{"q": "test"}'}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
AWS Bedrock format output::
|
||||||
|
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"toolUse": {
|
||||||
|
"toolUseId": "123",
|
||||||
|
"name": "search",
|
||||||
|
"input": {"q": "test"}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
"""
|
"""
|
||||||
if message["role"] == "tool":
|
if message["role"] == "tool":
|
||||||
# Try to parse the content as JSON if it looks like JSON
|
# Try to parse the content as JSON if it looks like JSON
|
||||||
|
|||||||
@@ -39,10 +39,8 @@ def get_presigned_url(
|
|||||||
|
|
||||||
Args:
|
Args:
|
||||||
region: AWS region for the service.
|
region: AWS region for the service.
|
||||||
credentials: Dictionary containing AWS credentials with keys:
|
credentials: Dictionary containing AWS credentials. Must include
|
||||||
- access_key: AWS access key ID
|
'access_key' and 'secret_key', with optional 'session_token'.
|
||||||
- secret_key: AWS secret access key
|
|
||||||
- session_token: AWS session token (optional)
|
|
||||||
language_code: Language code for transcription (e.g., "en-US").
|
language_code: Language code for transcription (e.g., "en-US").
|
||||||
media_encoding: Audio encoding format. Defaults to "pcm".
|
media_encoding: Audio encoding format. Defaults to "pcm".
|
||||||
sample_rate: Audio sample rate in Hz. Defaults to 16000.
|
sample_rate: Audio sample rate in Hz. Defaults to 16000.
|
||||||
@@ -325,9 +323,10 @@ def decode_event(message):
|
|||||||
message: Raw event stream message bytes received from AWS.
|
message: Raw event stream message bytes received from AWS.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Tuple containing:
|
A tuple of (headers, payload) where:
|
||||||
- Dictionary of parsed headers
|
|
||||||
- Dictionary of parsed JSON payload
|
- headers: Dictionary of parsed headers
|
||||||
|
- payload: Dictionary of parsed JSON payload
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
AssertionError: If CRC checksum verification fails.
|
AssertionError: If CRC checksum verification fails.
|
||||||
|
|||||||
@@ -764,21 +764,23 @@ class ElevenLabsHttpTTSService(WordTTSService):
|
|||||||
def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]:
|
def calculate_word_times(self, alignment_info: Mapping[str, Any]) -> List[Tuple[str, float]]:
|
||||||
"""Calculate word timing from character alignment data.
|
"""Calculate word timing from character alignment data.
|
||||||
|
|
||||||
Example input data:
|
|
||||||
{
|
|
||||||
"characters": [" ", "H", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d"],
|
|
||||||
"character_start_times_seconds": [0.0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
|
|
||||||
"character_end_times_seconds": [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
|
|
||||||
}
|
|
||||||
|
|
||||||
Would produce word times (with cumulative_time=0):
|
|
||||||
[("Hello", 0.1), ("world", 0.5)]
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
alignment_info: Character timing data from ElevenLabs.
|
alignment_info: Character timing data from ElevenLabs.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of (word, timestamp) pairs.
|
List of (word, timestamp) pairs.
|
||||||
|
|
||||||
|
Example input data::
|
||||||
|
|
||||||
|
{
|
||||||
|
"characters": [" ", "H", "e", "l", "l", "o", " ", "w", "o", "r", "l", "d"],
|
||||||
|
"character_start_times_seconds": [0.0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9],
|
||||||
|
"character_end_times_seconds": [0.1, 0.15, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0]
|
||||||
|
}
|
||||||
|
|
||||||
|
Would produce word times (with cumulative_time=0)::
|
||||||
|
|
||||||
|
[("Hello", 0.1), ("world", 0.5)]
|
||||||
"""
|
"""
|
||||||
chars = alignment_info.get("characters", [])
|
chars = alignment_info.get("characters", [])
|
||||||
char_start_times = alignment_info.get("character_start_times_seconds", [])
|
char_start_times = alignment_info.get("character_start_times_seconds", [])
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Google services module for Pipecat."""
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
from pipecat.services import DeprecatedModuleProxy
|
from pipecat.services import DeprecatedModuleProxy
|
||||||
|
|||||||
@@ -380,18 +380,48 @@ class GoogleLLMContext(OpenAILLMContext):
|
|||||||
System messages are stored separately and return None.
|
System messages are stored separately and return None.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
message: Message in standard format:
|
message: Message in standard format.
|
||||||
{
|
|
||||||
"role": "user/assistant/system/tool",
|
|
||||||
"content": str | [{"type": "text/image_url", ...}] | None,
|
|
||||||
"tool_calls": [{"function": {"name": str, "arguments": str}}]
|
|
||||||
}
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
Content object with:
|
Content object with role and parts, or None for system messages.
|
||||||
- role: "user" or "model" (converted from "assistant")
|
|
||||||
- parts: List[Part] containing text, inline_data, or function calls
|
Examples:
|
||||||
Returns None for system messages.
|
Standard text message::
|
||||||
|
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": "Hello there"
|
||||||
|
}
|
||||||
|
|
||||||
|
Converts to Google Content with::
|
||||||
|
|
||||||
|
Content(
|
||||||
|
role="user",
|
||||||
|
parts=[Part(text="Hello there")]
|
||||||
|
)
|
||||||
|
|
||||||
|
Standard function call message::
|
||||||
|
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"tool_calls": [
|
||||||
|
{
|
||||||
|
"function": {
|
||||||
|
"name": "search",
|
||||||
|
"arguments": '{"query": "test"}'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
Converts to Google Content with::
|
||||||
|
|
||||||
|
Content(
|
||||||
|
role="model",
|
||||||
|
parts=[Part(function_call=FunctionCall(name="search", args={"query": "test"}))]
|
||||||
|
)
|
||||||
|
|
||||||
|
System message returns None and stores content in self.system_message.
|
||||||
"""
|
"""
|
||||||
role = message["role"]
|
role = message["role"]
|
||||||
content = message.get("content", [])
|
content = message.get("content", [])
|
||||||
@@ -447,21 +477,73 @@ class GoogleLLMContext(OpenAILLMContext):
|
|||||||
Handles text, images, and function calls from Google's Content/Part objects.
|
Handles text, images, and function calls from Google's Content/Part objects.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
obj: Google Content object with:
|
obj: Google Content object with role and parts.
|
||||||
- role: "model" (converted to "assistant") or "user"
|
|
||||||
- parts: List[Part] containing text, inline_data, or function calls
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
List of messages in standard format:
|
List containing a single message in standard format.
|
||||||
[
|
|
||||||
{
|
Examples:
|
||||||
"role": "user/assistant/tool",
|
Google Content with text::
|
||||||
"content": [
|
|
||||||
{"type": "text", "text": str} |
|
Content(
|
||||||
{"type": "image_url", "image_url": {"url": str}}
|
role="user",
|
||||||
]
|
parts=[Part(text="Hello")]
|
||||||
}
|
)
|
||||||
]
|
|
||||||
|
Converts to::
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [{"type": "text", "text": "Hello"}]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
Google Content with function call::
|
||||||
|
|
||||||
|
Content(
|
||||||
|
role="model",
|
||||||
|
parts=[Part(function_call=FunctionCall(name="search", args={"q": "test"}))]
|
||||||
|
)
|
||||||
|
|
||||||
|
Converts to::
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"tool_calls": [
|
||||||
|
{
|
||||||
|
"id": "search",
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "search",
|
||||||
|
"arguments": '{"q": "test"}'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
Google Content with image::
|
||||||
|
|
||||||
|
Content(
|
||||||
|
role="user",
|
||||||
|
parts=[Part(inline_data=Blob(mime_type="image/jpeg", data=bytes_data))]
|
||||||
|
)
|
||||||
|
|
||||||
|
Converts to::
|
||||||
|
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "image_url",
|
||||||
|
"image_url": {"url": "data:image/jpeg;base64,<encoded_data>"}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
"""
|
"""
|
||||||
msg = {"role": obj.role, "content": []}
|
msg = {"role": obj.role, "content": []}
|
||||||
if msg["role"] == "model":
|
if msg["role"] == "model":
|
||||||
|
|||||||
@@ -471,8 +471,8 @@ class GoogleTTSService(TTSService):
|
|||||||
default application credentials (GOOGLE_APPLICATION_CREDENTIALS env var).
|
default application credentials (GOOGLE_APPLICATION_CREDENTIALS env var).
|
||||||
Only Chirp 3 HD and Journey voices are supported. Use GoogleHttpTTSService for other voices.
|
Only Chirp 3 HD and Journey voices are supported. Use GoogleHttpTTSService for other voices.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
```python
|
|
||||||
tts = GoogleTTSService(
|
tts = GoogleTTSService(
|
||||||
credentials_path="/path/to/service-account.json",
|
credentials_path="/path/to/service-account.json",
|
||||||
voice_id="en-US-Chirp3-HD-Charon",
|
voice_id="en-US-Chirp3-HD-Charon",
|
||||||
@@ -480,7 +480,6 @@ class GoogleTTSService(TTSService):
|
|||||||
language=Language.EN_US,
|
language=Language.EN_US,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
```
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
class InputParams(BaseModel):
|
class InputParams(BaseModel):
|
||||||
|
|||||||
@@ -128,13 +128,14 @@ class LLMService(AIService):
|
|||||||
parallel and sequential execution modes. Provides event handlers for
|
parallel and sequential execution modes. Provides event handlers for
|
||||||
completion timeouts and function call lifecycle events.
|
completion timeouts and function call lifecycle events.
|
||||||
|
|
||||||
Event handlers:
|
The service supports the following event handlers:
|
||||||
on_completion_timeout: Called when an LLM completion timeout occurs.
|
|
||||||
on_function_calls_started: Called when function calls are received and
|
- on_completion_timeout: Called when an LLM completion timeout occurs
|
||||||
execution is about to start.
|
- on_function_calls_started: Called when function calls are received and
|
||||||
|
execution is about to start
|
||||||
|
|
||||||
|
Example::
|
||||||
|
|
||||||
Example:
|
|
||||||
```python
|
|
||||||
@task.event_handler("on_completion_timeout")
|
@task.event_handler("on_completion_timeout")
|
||||||
async def on_completion_timeout(service):
|
async def on_completion_timeout(service):
|
||||||
logger.warning("LLM completion timed out")
|
logger.warning("LLM completion timed out")
|
||||||
@@ -142,7 +143,6 @@ class LLMService(AIService):
|
|||||||
@task.event_handler("on_function_calls_started")
|
@task.event_handler("on_function_calls_started")
|
||||||
async def on_function_calls_started(service, function_calls):
|
async def on_function_calls_started(service, function_calls):
|
||||||
logger.info(f"Starting {len(function_calls)} function calls")
|
logger.info(f"Starting {len(function_calls)} function calls")
|
||||||
```
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
|
# OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations.
|
||||||
|
|||||||
@@ -639,6 +639,7 @@ class OpenAIRealtimeBetaLLMService(LLMService):
|
|||||||
"""Maybe handle an error event related to retrieving a conversation item.
|
"""Maybe handle an error event related to retrieving a conversation item.
|
||||||
|
|
||||||
If the given error event is an error retrieving a conversation item:
|
If the given error event is an error retrieving a conversation item:
|
||||||
|
|
||||||
- set an exception on the future that retrieve_conversation_item() is waiting on
|
- set an exception on the future that retrieve_conversation_item() is waiting on
|
||||||
- return true
|
- return true
|
||||||
Otherwise:
|
Otherwise:
|
||||||
|
|||||||
@@ -59,8 +59,8 @@ class SarvamTTSService(TTSService):
|
|||||||
Indian languages. Provides control over voice characteristics like pitch, pace,
|
Indian languages. Provides control over voice characteristics like pitch, pace,
|
||||||
and loudness.
|
and loudness.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
```python
|
|
||||||
tts = SarvamTTSService(
|
tts = SarvamTTSService(
|
||||||
api_key="your-api-key",
|
api_key="your-api-key",
|
||||||
voice_id="anushka",
|
voice_id="anushka",
|
||||||
@@ -72,7 +72,6 @@ class SarvamTTSService(TTSService):
|
|||||||
pace=1.2
|
pace=1.2
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
```
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
class InputParams(BaseModel):
|
class InputParams(BaseModel):
|
||||||
|
|||||||
@@ -42,6 +42,7 @@ class TavusVideoService(AIService):
|
|||||||
are routed through Pipecat's media pipeline.
|
are routed through Pipecat's media pipeline.
|
||||||
|
|
||||||
In use cases with DailyTransport, this creates two distinct virtual rooms:
|
In use cases with DailyTransport, this creates two distinct virtual rooms:
|
||||||
|
|
||||||
- Tavus room: Contains the Tavus Avatar and the Pipecat Bot
|
- Tavus room: Contains the Tavus Avatar and the Pipecat Bot
|
||||||
- User room: Contains the Pipecat Bot and the user
|
- User room: Contains the Pipecat Bot and the user
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -549,12 +549,11 @@ class WebsocketTTSService(TTSService, WebsocketService):
|
|||||||
Event handlers:
|
Event handlers:
|
||||||
on_connection_error: Called when a websocket connection error occurs.
|
on_connection_error: Called when a websocket connection error occurs.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
```python
|
|
||||||
@tts.event_handler("on_connection_error")
|
@tts.event_handler("on_connection_error")
|
||||||
async def on_connection_error(tts: TTSService, error: str):
|
async def on_connection_error(tts: TTSService, error: str):
|
||||||
logger.error(f"TTS connection error: {error}")
|
logger.error(f"TTS connection error: {error}")
|
||||||
```
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
|
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
|
||||||
@@ -622,12 +621,11 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService):
|
|||||||
Event handlers:
|
Event handlers:
|
||||||
on_connection_error: Called when a websocket connection error occurs.
|
on_connection_error: Called when a websocket connection error occurs.
|
||||||
|
|
||||||
Example:
|
Example::
|
||||||
```python
|
|
||||||
@tts.event_handler("on_connection_error")
|
@tts.event_handler("on_connection_error")
|
||||||
async def on_connection_error(tts: TTSService, error: str):
|
async def on_connection_error(tts: TTSService, error: str):
|
||||||
logger.error(f"TTS connection error: {error}")
|
logger.error(f"TTS connection error: {error}")
|
||||||
```
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
|
def __init__(self, *, reconnect_on_error: bool = True, **kwargs):
|
||||||
|
|||||||
@@ -4,14 +4,33 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base notifier interface for Pipecat."""
|
||||||
|
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
|
||||||
class BaseNotifier(ABC):
|
class BaseNotifier(ABC):
|
||||||
|
"""Abstract base class for notification mechanisms.
|
||||||
|
|
||||||
|
Provides a standard interface for implementing notification and waiting
|
||||||
|
patterns used for event coordination and signaling between components
|
||||||
|
in the Pipecat framework.
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def notify(self):
|
async def notify(self):
|
||||||
|
"""Send a notification signal.
|
||||||
|
|
||||||
|
Implementations should trigger any waiting coroutines or processes
|
||||||
|
that are blocked on this notifier.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def wait(self):
|
async def wait(self):
|
||||||
|
"""Wait for a notification signal.
|
||||||
|
|
||||||
|
Implementations should block until a notification is received
|
||||||
|
from the corresponding notify() call.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,18 +4,42 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Event-based notifier implementation using asyncio Event primitives."""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
|
||||||
from pipecat.sync.base_notifier import BaseNotifier
|
from pipecat.sync.base_notifier import BaseNotifier
|
||||||
|
|
||||||
|
|
||||||
class EventNotifier(BaseNotifier):
|
class EventNotifier(BaseNotifier):
|
||||||
|
"""Event-based notifier using asyncio.Event for task synchronization.
|
||||||
|
|
||||||
|
Provides a simple notification mechanism where one task can signal
|
||||||
|
an event and other tasks can wait for that event to occur. The event
|
||||||
|
is automatically cleared after each wait operation.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
"""Initialize the event notifier.
|
||||||
|
|
||||||
|
Creates an internal asyncio.Event for managing notifications.
|
||||||
|
"""
|
||||||
self._event = asyncio.Event()
|
self._event = asyncio.Event()
|
||||||
|
|
||||||
async def notify(self):
|
async def notify(self):
|
||||||
|
"""Signal the event to notify waiting tasks.
|
||||||
|
|
||||||
|
Sets the internal event, causing any tasks waiting on this
|
||||||
|
notifier to be awakened.
|
||||||
|
"""
|
||||||
self._event.set()
|
self._event.set()
|
||||||
|
|
||||||
async def wait(self):
|
async def wait(self):
|
||||||
|
"""Wait for the event to be signaled.
|
||||||
|
|
||||||
|
Blocks until another task calls notify(). Automatically clears
|
||||||
|
the event after being awakened so subsequent calls will wait
|
||||||
|
for the next notification.
|
||||||
|
"""
|
||||||
await self._event.wait()
|
await self._event.wait()
|
||||||
self._event.clear()
|
self._event.clear()
|
||||||
|
|||||||
@@ -4,6 +4,8 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Testing utilities for Pipecat pipeline components."""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple
|
from typing import Any, Awaitable, Callable, Dict, List, Optional, Sequence, Tuple
|
||||||
@@ -24,15 +26,27 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class SleepFrame(SystemFrame):
|
class SleepFrame(SystemFrame):
|
||||||
"""This frame is used by test framework to introduce some sleep time before
|
"""A system frame that introduces a sleep delay in the test pipeline.
|
||||||
the next frame is pushed. This is useful to control system frames vs data or
|
|
||||||
control frames.
|
This frame is used by the test framework to control timing between
|
||||||
|
frame processing, allowing tests to separate system frames from
|
||||||
|
data or control frames.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
sleep: Duration to sleep in seconds before processing the next frame.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
sleep: float = 0.1
|
sleep: float = 0.1
|
||||||
|
|
||||||
|
|
||||||
class HeartbeatsObserver(BaseObserver):
|
class HeartbeatsObserver(BaseObserver):
|
||||||
|
"""Observer that monitors heartbeat frames from a specific processor.
|
||||||
|
|
||||||
|
This observer watches for HeartbeatFrames from a target processor and
|
||||||
|
invokes a callback when they are detected, useful for testing timing
|
||||||
|
and lifecycle events.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -40,11 +54,23 @@ class HeartbeatsObserver(BaseObserver):
|
|||||||
heartbeat_callback: Callable[[FrameProcessor, HeartbeatFrame], Awaitable[None]],
|
heartbeat_callback: Callable[[FrameProcessor, HeartbeatFrame], Awaitable[None]],
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the heartbeats observer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
target: The frame processor to monitor for heartbeat frames.
|
||||||
|
heartbeat_callback: Async callback function to invoke when heartbeats are detected.
|
||||||
|
**kwargs: Additional arguments passed to the parent observer.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._target = target
|
self._target = target
|
||||||
self._callback = heartbeat_callback
|
self._callback = heartbeat_callback
|
||||||
|
|
||||||
async def on_push_frame(self, data: FramePushed):
|
async def on_push_frame(self, data: FramePushed):
|
||||||
|
"""Handle frame push events and detect heartbeats from target processor.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: The frame push event data containing source and frame information.
|
||||||
|
"""
|
||||||
src = data.source
|
src = data.source
|
||||||
frame = data.frame
|
frame = data.frame
|
||||||
|
|
||||||
@@ -53,6 +79,13 @@ class HeartbeatsObserver(BaseObserver):
|
|||||||
|
|
||||||
|
|
||||||
class QueuedFrameProcessor(FrameProcessor):
|
class QueuedFrameProcessor(FrameProcessor):
|
||||||
|
"""A processor that captures frames in a queue for testing purposes.
|
||||||
|
|
||||||
|
This processor intercepts frames flowing in a specific direction and
|
||||||
|
stores them in a queue for later inspection during testing, while
|
||||||
|
still allowing the frames to continue through the pipeline.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -60,12 +93,25 @@ class QueuedFrameProcessor(FrameProcessor):
|
|||||||
queue_direction: FrameDirection,
|
queue_direction: FrameDirection,
|
||||||
ignore_start: bool = True,
|
ignore_start: bool = True,
|
||||||
):
|
):
|
||||||
|
"""Initialize the queued frame processor.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
queue: The asyncio queue to store captured frames.
|
||||||
|
queue_direction: The direction of frames to capture (UPSTREAM or DOWNSTREAM).
|
||||||
|
ignore_start: Whether to ignore StartFrames when capturing.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._queue = queue
|
self._queue = queue
|
||||||
self._queue_direction = queue_direction
|
self._queue_direction = queue_direction
|
||||||
self._ignore_start = ignore_start
|
self._ignore_start = ignore_start
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and capture them in the queue if they match the direction.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction the frame is flowing.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
if direction == self._queue_direction:
|
if direction == self._queue_direction:
|
||||||
@@ -85,6 +131,28 @@ async def run_test(
|
|||||||
start_metadata: Optional[Dict[str, Any]] = None,
|
start_metadata: Optional[Dict[str, Any]] = None,
|
||||||
send_end_frame: bool = True,
|
send_end_frame: bool = True,
|
||||||
) -> Tuple[Sequence[Frame], Sequence[Frame]]:
|
) -> Tuple[Sequence[Frame], Sequence[Frame]]:
|
||||||
|
"""Run a test pipeline with the specified processor and validate frame flow.
|
||||||
|
|
||||||
|
This function creates a test pipeline with the given processor, sends the
|
||||||
|
specified frames through it, and validates that the expected frames are
|
||||||
|
received in both upstream and downstream directions.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
processor: The frame processor to test.
|
||||||
|
frames_to_send: Sequence of frames to send through the processor.
|
||||||
|
expected_down_frames: Expected frame types flowing downstream (optional).
|
||||||
|
expected_up_frames: Expected frame types flowing upstream (optional).
|
||||||
|
ignore_start: Whether to ignore StartFrames in frame validation.
|
||||||
|
observers: Optional list of observers to attach to the pipeline.
|
||||||
|
start_metadata: Optional metadata to include with the StartFrame.
|
||||||
|
send_end_frame: Whether to send an EndFrame at the end of the test.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Tuple containing (downstream_frames, upstream_frames) that were received.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
AssertionError: If the received frames don't match the expected frame types.
|
||||||
|
"""
|
||||||
observers = observers or []
|
observers = observers or []
|
||||||
start_metadata = start_metadata or {}
|
start_metadata = start_metadata or {}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base input transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides the BaseInputTransport class which handles audio and video
|
||||||
|
input processing, including VAD, turn analysis, and interruption management.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -47,7 +53,20 @@ AUDIO_INPUT_TIMEOUT_SECS = 0.5
|
|||||||
|
|
||||||
|
|
||||||
class BaseInputTransport(FrameProcessor):
|
class BaseInputTransport(FrameProcessor):
|
||||||
|
"""Base class for input transport implementations.
|
||||||
|
|
||||||
|
Handles audio and video input processing including Voice Activity Detection,
|
||||||
|
turn analysis, audio filtering, and user interaction management. Supports
|
||||||
|
interruption handling and provides hooks for transport-specific implementations.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, params: TransportParams, **kwargs):
|
def __init__(self, params: TransportParams, **kwargs):
|
||||||
|
"""Initialize the base input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -115,25 +134,54 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
self._params.video_out_color_format = self._params.camera_out_color_format
|
self._params.video_out_color_format = self._params.camera_out_color_format
|
||||||
|
|
||||||
def enable_audio_in_stream_on_start(self, enabled: bool) -> None:
|
def enable_audio_in_stream_on_start(self, enabled: bool) -> None:
|
||||||
|
"""Enable or disable audio streaming on transport start.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
enabled: Whether to start audio streaming immediately on transport start.
|
||||||
|
"""
|
||||||
logger.debug(f"Enabling audio on start. {enabled}")
|
logger.debug(f"Enabling audio on start. {enabled}")
|
||||||
self._params.audio_in_stream_on_start = enabled
|
self._params.audio_in_stream_on_start = enabled
|
||||||
|
|
||||||
async def start_audio_in_streaming(self):
|
async def start_audio_in_streaming(self):
|
||||||
|
"""Start audio input streaming.
|
||||||
|
|
||||||
|
Override in subclasses to implement transport-specific audio streaming.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def sample_rate(self) -> int:
|
def sample_rate(self) -> int:
|
||||||
|
"""Get the current audio sample rate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The sample rate in Hz.
|
||||||
|
"""
|
||||||
return self._sample_rate
|
return self._sample_rate
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def vad_analyzer(self) -> Optional[VADAnalyzer]:
|
def vad_analyzer(self) -> Optional[VADAnalyzer]:
|
||||||
|
"""Get the Voice Activity Detection analyzer.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The VAD analyzer instance if configured, None otherwise.
|
||||||
|
"""
|
||||||
return self._params.vad_analyzer
|
return self._params.vad_analyzer
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def turn_analyzer(self) -> Optional[BaseTurnAnalyzer]:
|
def turn_analyzer(self) -> Optional[BaseTurnAnalyzer]:
|
||||||
|
"""Get the turn-taking analyzer.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The turn analyzer instance if configured, None otherwise.
|
||||||
|
"""
|
||||||
return self._params.turn_analyzer
|
return self._params.turn_analyzer
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the input transport and initialize components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
self._paused = False
|
self._paused = False
|
||||||
self._user_speaking = False
|
self._user_speaking = False
|
||||||
|
|
||||||
@@ -152,6 +200,11 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
await self._params.audio_in_filter.start(self._sample_rate)
|
await self._params.audio_in_filter.start(self._sample_rate)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the input transport and cleanup resources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
# Cancel and wait for the audio input task to finish.
|
# Cancel and wait for the audio input task to finish.
|
||||||
await self._cancel_audio_task()
|
await self._cancel_audio_task()
|
||||||
# Stop audio filter.
|
# Stop audio filter.
|
||||||
@@ -159,6 +212,11 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
await self._params.audio_in_filter.stop()
|
await self._params.audio_in_filter.stop()
|
||||||
|
|
||||||
async def pause(self, frame: StopFrame):
|
async def pause(self, frame: StopFrame):
|
||||||
|
"""Pause the input transport temporarily.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The stop frame signaling transport pause.
|
||||||
|
"""
|
||||||
self._paused = True
|
self._paused = True
|
||||||
# Cancel task so we clear the queue
|
# Cancel task so we clear the queue
|
||||||
await self._cancel_audio_task()
|
await self._cancel_audio_task()
|
||||||
@@ -166,19 +224,38 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
self._create_audio_task()
|
self._create_audio_task()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the input transport and stop all processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
# Cancel and wait for the audio input task to finish.
|
# Cancel and wait for the audio input task to finish.
|
||||||
await self._cancel_audio_task()
|
await self._cancel_audio_task()
|
||||||
|
|
||||||
async def set_transport_ready(self, frame: StartFrame):
|
async def set_transport_ready(self, frame: StartFrame):
|
||||||
"""To be called when the transport is ready to stream."""
|
"""Called when the transport is ready to stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
# Create audio input queue and task if needed.
|
# Create audio input queue and task if needed.
|
||||||
self._create_audio_task()
|
self._create_audio_task()
|
||||||
|
|
||||||
async def push_video_frame(self, frame: InputImageRawFrame):
|
async def push_video_frame(self, frame: InputImageRawFrame):
|
||||||
|
"""Push a video frame downstream if video input is enabled.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The input video frame to process.
|
||||||
|
"""
|
||||||
if self._params.video_in_enabled and not self._paused:
|
if self._params.video_in_enabled and not self._paused:
|
||||||
await self.push_frame(frame)
|
await self.push_frame(frame)
|
||||||
|
|
||||||
async def push_audio_frame(self, frame: InputAudioRawFrame):
|
async def push_audio_frame(self, frame: InputAudioRawFrame):
|
||||||
|
"""Push an audio frame to the processing queue if audio input is enabled.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The input audio frame to process.
|
||||||
|
"""
|
||||||
if self._params.audio_in_enabled and not self._paused:
|
if self._params.audio_in_enabled and not self._paused:
|
||||||
await self._audio_in_queue.put(frame)
|
await self._audio_in_queue.put(frame)
|
||||||
|
|
||||||
@@ -187,6 +264,12 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process incoming frames and handle transport-specific logic.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow in the pipeline.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
# Specific system frames
|
# Specific system frames
|
||||||
@@ -238,12 +321,14 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
async def _handle_bot_interruption(self, frame: BotInterruptionFrame):
|
async def _handle_bot_interruption(self, frame: BotInterruptionFrame):
|
||||||
|
"""Handle bot interruption frames."""
|
||||||
logger.debug("Bot interruption")
|
logger.debug("Bot interruption")
|
||||||
if self.interruptions_allowed:
|
if self.interruptions_allowed:
|
||||||
await self._start_interruption()
|
await self._start_interruption()
|
||||||
await self.push_frame(StartInterruptionFrame())
|
await self.push_frame(StartInterruptionFrame())
|
||||||
|
|
||||||
async def _handle_user_interruption(self, frame: Frame):
|
async def _handle_user_interruption(self, frame: Frame):
|
||||||
|
"""Handle user interruption events based on speaking state."""
|
||||||
if isinstance(frame, UserStartedSpeakingFrame):
|
if isinstance(frame, UserStartedSpeakingFrame):
|
||||||
logger.debug("User started speaking")
|
logger.debug("User started speaking")
|
||||||
self._user_speaking = True
|
self._user_speaking = True
|
||||||
@@ -281,9 +366,11 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
async def _handle_bot_started_speaking(self, frame: BotStartedSpeakingFrame):
|
async def _handle_bot_started_speaking(self, frame: BotStartedSpeakingFrame):
|
||||||
|
"""Update bot speaking state when bot starts speaking."""
|
||||||
self._bot_speaking = True
|
self._bot_speaking = True
|
||||||
|
|
||||||
async def _handle_bot_stopped_speaking(self, frame: BotStoppedSpeakingFrame):
|
async def _handle_bot_stopped_speaking(self, frame: BotStoppedSpeakingFrame):
|
||||||
|
"""Update bot speaking state when bot stops speaking."""
|
||||||
self._bot_speaking = False
|
self._bot_speaking = False
|
||||||
|
|
||||||
#
|
#
|
||||||
@@ -291,16 +378,19 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def _create_audio_task(self):
|
def _create_audio_task(self):
|
||||||
|
"""Create the audio processing task if audio input is enabled."""
|
||||||
if not self._audio_task and self._params.audio_in_enabled:
|
if not self._audio_task and self._params.audio_in_enabled:
|
||||||
self._audio_in_queue = asyncio.Queue()
|
self._audio_in_queue = asyncio.Queue()
|
||||||
self._audio_task = self.create_task(self._audio_task_handler())
|
self._audio_task = self.create_task(self._audio_task_handler())
|
||||||
|
|
||||||
async def _cancel_audio_task(self):
|
async def _cancel_audio_task(self):
|
||||||
|
"""Cancel and cleanup the audio processing task."""
|
||||||
if self._audio_task:
|
if self._audio_task:
|
||||||
await self.cancel_task(self._audio_task)
|
await self.cancel_task(self._audio_task)
|
||||||
self._audio_task = None
|
self._audio_task = None
|
||||||
|
|
||||||
async def _vad_analyze(self, audio_frame: InputAudioRawFrame) -> VADState:
|
async def _vad_analyze(self, audio_frame: InputAudioRawFrame) -> VADState:
|
||||||
|
"""Analyze audio frame for voice activity."""
|
||||||
state = VADState.QUIET
|
state = VADState.QUIET
|
||||||
if self.vad_analyzer:
|
if self.vad_analyzer:
|
||||||
state = await self.get_event_loop().run_in_executor(
|
state = await self.get_event_loop().run_in_executor(
|
||||||
@@ -309,6 +399,7 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
return state
|
return state
|
||||||
|
|
||||||
async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState):
|
async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState):
|
||||||
|
"""Handle Voice Activity Detection results and generate appropriate frames."""
|
||||||
new_vad_state = await self._vad_analyze(audio_frame)
|
new_vad_state = await self._vad_analyze(audio_frame)
|
||||||
if (
|
if (
|
||||||
new_vad_state != vad_state
|
new_vad_state != vad_state
|
||||||
@@ -339,18 +430,21 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
return vad_state
|
return vad_state
|
||||||
|
|
||||||
async def _handle_end_of_turn(self):
|
async def _handle_end_of_turn(self):
|
||||||
|
"""Handle end-of-turn analysis and generate prediction results."""
|
||||||
if self.turn_analyzer:
|
if self.turn_analyzer:
|
||||||
state, prediction = await self.turn_analyzer.analyze_end_of_turn()
|
state, prediction = await self.turn_analyzer.analyze_end_of_turn()
|
||||||
await self._handle_prediction_result(prediction)
|
await self._handle_prediction_result(prediction)
|
||||||
await self._handle_end_of_turn_complete(state)
|
await self._handle_end_of_turn_complete(state)
|
||||||
|
|
||||||
async def _handle_end_of_turn_complete(self, state: EndOfTurnState):
|
async def _handle_end_of_turn_complete(self, state: EndOfTurnState):
|
||||||
|
"""Handle completion of end-of-turn analysis."""
|
||||||
if state == EndOfTurnState.COMPLETE:
|
if state == EndOfTurnState.COMPLETE:
|
||||||
await self._handle_user_interruption(UserStoppedSpeakingFrame())
|
await self._handle_user_interruption(UserStoppedSpeakingFrame())
|
||||||
|
|
||||||
async def _run_turn_analyzer(
|
async def _run_turn_analyzer(
|
||||||
self, frame: InputAudioRawFrame, vad_state: VADState, previous_vad_state: VADState
|
self, frame: InputAudioRawFrame, vad_state: VADState, previous_vad_state: VADState
|
||||||
):
|
):
|
||||||
|
"""Run turn analysis on audio frame and handle results."""
|
||||||
is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
|
is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
|
||||||
# If silence exceeds threshold, we are going to receive EndOfTurnState.COMPLETE
|
# If silence exceeds threshold, we are going to receive EndOfTurnState.COMPLETE
|
||||||
end_of_turn_state = self._params.turn_analyzer.append_audio(frame.audio, is_speech)
|
end_of_turn_state = self._params.turn_analyzer.append_audio(frame.audio, is_speech)
|
||||||
@@ -361,6 +455,7 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
await self._handle_end_of_turn()
|
await self._handle_end_of_turn()
|
||||||
|
|
||||||
async def _audio_task_handler(self):
|
async def _audio_task_handler(self):
|
||||||
|
"""Main audio processing task handler for VAD and turn analysis."""
|
||||||
vad_state: VADState = VADState.QUIET
|
vad_state: VADState = VADState.QUIET
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
@@ -399,9 +494,5 @@ class BaseInputTransport(FrameProcessor):
|
|||||||
self.reset_watchdog()
|
self.reset_watchdog()
|
||||||
|
|
||||||
async def _handle_prediction_result(self, result: MetricsData):
|
async def _handle_prediction_result(self, result: MetricsData):
|
||||||
"""Handle a prediction result event from the turn analyzer.
|
"""Handle a prediction result event from the turn analyzer."""
|
||||||
|
|
||||||
Args:
|
|
||||||
result: The prediction result MetricsData.
|
|
||||||
"""
|
|
||||||
await self.push_frame(MetricsFrame(data=[result]))
|
await self.push_frame(MetricsFrame(data=[result]))
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base output transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides the BaseOutputTransport class which handles audio and video
|
||||||
|
output processing, including frame buffering, mixing, timing, and media streaming.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import itertools
|
import itertools
|
||||||
import sys
|
import sys
|
||||||
@@ -46,7 +52,20 @@ BOT_VAD_STOP_SECS = 0.35
|
|||||||
|
|
||||||
|
|
||||||
class BaseOutputTransport(FrameProcessor):
|
class BaseOutputTransport(FrameProcessor):
|
||||||
|
"""Base class for output transport implementations.
|
||||||
|
|
||||||
|
Handles audio and video output processing including frame buffering, audio mixing,
|
||||||
|
timing coordination, and media streaming. Supports multiple output destinations
|
||||||
|
and provides interruption handling for real-time communication.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, params: TransportParams, **kwargs):
|
def __init__(self, params: TransportParams, **kwargs):
|
||||||
|
"""Initialize the base output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
|
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -67,13 +86,28 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def sample_rate(self) -> int:
|
def sample_rate(self) -> int:
|
||||||
|
"""Get the current audio sample rate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The sample rate in Hz.
|
||||||
|
"""
|
||||||
return self._sample_rate
|
return self._sample_rate
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def audio_chunk_size(self) -> int:
|
def audio_chunk_size(self) -> int:
|
||||||
|
"""Get the audio chunk size for output processing.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The size of audio chunks in bytes.
|
||||||
|
"""
|
||||||
return self._audio_chunk_size
|
return self._audio_chunk_size
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the output transport and initialize components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
|
self._sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
|
||||||
|
|
||||||
# We will write 10ms*CHUNKS of audio at a time (where CHUNKS is the
|
# We will write 10ms*CHUNKS of audio at a time (where CHUNKS is the
|
||||||
@@ -83,15 +117,29 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
self._audio_chunk_size = audio_bytes_10ms * self._params.audio_out_10ms_chunks
|
self._audio_chunk_size = audio_bytes_10ms * self._params.audio_out_10ms_chunks
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the output transport and cleanup resources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
for _, sender in self._media_senders.items():
|
for _, sender in self._media_senders.items():
|
||||||
await sender.stop(frame)
|
await sender.stop(frame)
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the output transport and stop all processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
for _, sender in self._media_senders.items():
|
for _, sender in self._media_senders.items():
|
||||||
await sender.cancel(frame)
|
await sender.cancel(frame)
|
||||||
|
|
||||||
async def set_transport_ready(self, frame: StartFrame):
|
async def set_transport_ready(self, frame: StartFrame):
|
||||||
"""To be called when the transport is ready to stream."""
|
"""Called when the transport is ready to stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
# Register destinations.
|
# Register destinations.
|
||||||
for destination in self._params.audio_out_destinations:
|
for destination in self._params.audio_out_destinations:
|
||||||
await self.register_audio_destination(destination)
|
await self.register_audio_destination(destination)
|
||||||
@@ -127,27 +175,67 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._media_senders[destination].start(frame)
|
await self._media_senders[destination].start(frame)
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a transport message.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The transport message frame to send.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def register_video_destination(self, destination: str):
|
async def register_video_destination(self, destination: str):
|
||||||
|
"""Register a video output destination.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
destination: The destination identifier to register.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def register_audio_destination(self, destination: str):
|
async def register_audio_destination(self, destination: str):
|
||||||
|
"""Register an audio output destination.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
destination: The destination identifier to register.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def write_video_frame(self, frame: OutputImageRawFrame):
|
async def write_video_frame(self, frame: OutputImageRawFrame):
|
||||||
|
"""Write a video frame to the transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output video frame to write.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output audio frame to write.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def write_dtmf(self, frame: OutputDTMFFrame | OutputDTMFUrgentFrame):
|
async def write_dtmf(self, frame: OutputDTMFFrame | OutputDTMFUrgentFrame):
|
||||||
|
"""Write a DTMF tone to the transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The DTMF frame to write.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def send_audio(self, frame: OutputAudioRawFrame):
|
async def send_audio(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Send an audio frame downstream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to send.
|
||||||
|
"""
|
||||||
await self.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
await self.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
||||||
|
|
||||||
async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
|
async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
|
||||||
|
"""Send an image frame downstream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The image frame to send.
|
||||||
|
"""
|
||||||
await self.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
await self.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
||||||
|
|
||||||
#
|
#
|
||||||
@@ -155,6 +243,12 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process incoming frames and handle transport-specific logic.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow in the pipeline.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
#
|
#
|
||||||
@@ -200,6 +294,7 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._handle_frame(frame)
|
await self._handle_frame(frame)
|
||||||
|
|
||||||
async def _handle_frame(self, frame: Frame):
|
async def _handle_frame(self, frame: Frame):
|
||||||
|
"""Handle frames by routing them to appropriate media senders."""
|
||||||
if frame.transport_destination not in self._media_senders:
|
if frame.transport_destination not in self._media_senders:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
f"{self} destination [{frame.transport_destination}] not registered for frame {frame}"
|
f"{self} destination [{frame.transport_destination}] not registered for frame {frame}"
|
||||||
@@ -226,6 +321,12 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
class MediaSender:
|
class MediaSender:
|
||||||
|
"""Handles media streaming for a specific destination.
|
||||||
|
|
||||||
|
Manages audio and video output processing including buffering, timing,
|
||||||
|
mixing, and frame delivery for a single output destination.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: "BaseOutputTransport",
|
transport: "BaseOutputTransport",
|
||||||
@@ -235,6 +336,15 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
audio_chunk_size: int,
|
audio_chunk_size: int,
|
||||||
params: TransportParams,
|
params: TransportParams,
|
||||||
):
|
):
|
||||||
|
"""Initialize the media sender.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
destination: The destination identifier for this sender.
|
||||||
|
sample_rate: The audio sample rate in Hz.
|
||||||
|
audio_chunk_size: The size of audio chunks in bytes.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
"""
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._destination = destination
|
self._destination = destination
|
||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
@@ -266,13 +376,28 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def sample_rate(self) -> int:
|
def sample_rate(self) -> int:
|
||||||
|
"""Get the audio sample rate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The sample rate in Hz.
|
||||||
|
"""
|
||||||
return self._sample_rate
|
return self._sample_rate
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def audio_chunk_size(self) -> int:
|
def audio_chunk_size(self) -> int:
|
||||||
|
"""Get the audio chunk size.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The size of audio chunks in bytes.
|
||||||
|
"""
|
||||||
return self._audio_chunk_size
|
return self._audio_chunk_size
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the media sender and initialize components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
self._audio_buffer = bytearray()
|
self._audio_buffer = bytearray()
|
||||||
|
|
||||||
# Create all tasks.
|
# Create all tasks.
|
||||||
@@ -293,6 +418,11 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._mixer.start(self._sample_rate)
|
await self._mixer.start(self._sample_rate)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the media sender and cleanup resources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling sender shutdown.
|
||||||
|
"""
|
||||||
# Let the sink tasks process the queue until they reach this EndFrame.
|
# Let the sink tasks process the queue until they reach this EndFrame.
|
||||||
await self._clock_queue.put((sys.maxsize, frame.id, frame))
|
await self._clock_queue.put((sys.maxsize, frame.id, frame))
|
||||||
await self._audio_queue.put(frame)
|
await self._audio_queue.put(frame)
|
||||||
@@ -314,12 +444,22 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._cancel_video_task()
|
await self._cancel_video_task()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the media sender and stop all processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
# Since we are cancelling everything it doesn't matter what task we cancel first.
|
# Since we are cancelling everything it doesn't matter what task we cancel first.
|
||||||
await self._cancel_audio_task()
|
await self._cancel_audio_task()
|
||||||
await self._cancel_clock_task()
|
await self._cancel_clock_task()
|
||||||
await self._cancel_video_task()
|
await self._cancel_video_task()
|
||||||
|
|
||||||
async def handle_interruptions(self, _: StartInterruptionFrame):
|
async def handle_interruptions(self, _: StartInterruptionFrame):
|
||||||
|
"""Handle interruption events by restarting tasks and clearing buffers.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
_: The start interruption frame (unused).
|
||||||
|
"""
|
||||||
if not self._transport.interruptions_allowed:
|
if not self._transport.interruptions_allowed:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -335,6 +475,11 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._bot_stopped_speaking()
|
await self._bot_stopped_speaking()
|
||||||
|
|
||||||
async def handle_audio_frame(self, frame: OutputAudioRawFrame):
|
async def handle_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Handle incoming audio frames by buffering and chunking.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output audio frame to handle.
|
||||||
|
"""
|
||||||
if not self._params.audio_out_enabled:
|
if not self._params.audio_out_enabled:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -357,6 +502,11 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
|
self._audio_buffer = self._audio_buffer[self._audio_chunk_size :]
|
||||||
|
|
||||||
async def handle_image_frame(self, frame: OutputImageRawFrame | SpriteFrame):
|
async def handle_image_frame(self, frame: OutputImageRawFrame | SpriteFrame):
|
||||||
|
"""Handle incoming image frames for video output.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output image or sprite frame to handle.
|
||||||
|
"""
|
||||||
if not self._params.video_out_enabled:
|
if not self._params.video_out_enabled:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -368,12 +518,27 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._set_video_images(frame.images)
|
await self._set_video_images(frame.images)
|
||||||
|
|
||||||
async def handle_timed_frame(self, frame: Frame):
|
async def handle_timed_frame(self, frame: Frame):
|
||||||
|
"""Handle frames with presentation timestamps.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame with timing information to handle.
|
||||||
|
"""
|
||||||
await self._clock_queue.put((frame.pts, frame.id, frame))
|
await self._clock_queue.put((frame.pts, frame.id, frame))
|
||||||
|
|
||||||
async def handle_sync_frame(self, frame: Frame):
|
async def handle_sync_frame(self, frame: Frame):
|
||||||
|
"""Handle frames that need synchronized processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to handle synchronously.
|
||||||
|
"""
|
||||||
await self._audio_queue.put(frame)
|
await self._audio_queue.put(frame)
|
||||||
|
|
||||||
async def handle_mixer_control_frame(self, frame: MixerControlFrame):
|
async def handle_mixer_control_frame(self, frame: MixerControlFrame):
|
||||||
|
"""Handle audio mixer control frames.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The mixer control frame to handle.
|
||||||
|
"""
|
||||||
if self._mixer:
|
if self._mixer:
|
||||||
await self._mixer.process_frame(frame)
|
await self._mixer.process_frame(frame)
|
||||||
|
|
||||||
@@ -382,16 +547,19 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def _create_audio_task(self):
|
def _create_audio_task(self):
|
||||||
|
"""Create the audio processing task."""
|
||||||
if not self._audio_task:
|
if not self._audio_task:
|
||||||
self._audio_queue = asyncio.Queue()
|
self._audio_queue = asyncio.Queue()
|
||||||
self._audio_task = self._transport.create_task(self._audio_task_handler())
|
self._audio_task = self._transport.create_task(self._audio_task_handler())
|
||||||
|
|
||||||
async def _cancel_audio_task(self):
|
async def _cancel_audio_task(self):
|
||||||
|
"""Cancel and cleanup the audio processing task."""
|
||||||
if self._audio_task:
|
if self._audio_task:
|
||||||
await self._transport.cancel_task(self._audio_task)
|
await self._transport.cancel_task(self._audio_task)
|
||||||
self._audio_task = None
|
self._audio_task = None
|
||||||
|
|
||||||
async def _bot_started_speaking(self):
|
async def _bot_started_speaking(self):
|
||||||
|
"""Handle bot started speaking event."""
|
||||||
if not self._bot_speaking:
|
if not self._bot_speaking:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking"
|
f"Bot{f' [{self._destination}]' if self._destination else ''} started speaking"
|
||||||
@@ -407,6 +575,7 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
self._bot_speaking = True
|
self._bot_speaking = True
|
||||||
|
|
||||||
async def _bot_stopped_speaking(self):
|
async def _bot_stopped_speaking(self):
|
||||||
|
"""Handle bot stopped speaking event."""
|
||||||
if self._bot_speaking:
|
if self._bot_speaking:
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking"
|
f"Bot{f' [{self._destination}]' if self._destination else ''} stopped speaking"
|
||||||
@@ -426,6 +595,11 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
self._audio_buffer = bytearray()
|
self._audio_buffer = bytearray()
|
||||||
|
|
||||||
async def _handle_frame(self, frame: Frame):
|
async def _handle_frame(self, frame: Frame):
|
||||||
|
"""Handle various frame types with appropriate processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to handle.
|
||||||
|
"""
|
||||||
if isinstance(frame, OutputImageRawFrame):
|
if isinstance(frame, OutputImageRawFrame):
|
||||||
await self._set_video_image(frame)
|
await self._set_video_image(frame)
|
||||||
elif isinstance(frame, SpriteFrame):
|
elif isinstance(frame, SpriteFrame):
|
||||||
@@ -436,6 +610,12 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await self._transport.write_dtmf(frame)
|
await self._transport.write_dtmf(frame)
|
||||||
|
|
||||||
def _next_frame(self) -> AsyncGenerator[Frame, None]:
|
def _next_frame(self) -> AsyncGenerator[Frame, None]:
|
||||||
|
"""Generate the next frame for audio processing.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
An async generator yielding frames for processing.
|
||||||
|
"""
|
||||||
|
|
||||||
async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
|
async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]:
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
@@ -480,6 +660,7 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
return without_mixer(BOT_VAD_STOP_SECS)
|
return without_mixer(BOT_VAD_STOP_SECS)
|
||||||
|
|
||||||
async def _audio_task_handler(self):
|
async def _audio_task_handler(self):
|
||||||
|
"""Main audio processing task handler."""
|
||||||
# Push a BotSpeakingFrame every 200ms, we don't really need to push it
|
# Push a BotSpeakingFrame every 200ms, we don't really need to push it
|
||||||
# at every audio chunk. If the audio chunk is bigger than 200ms, push at
|
# at every audio chunk. If the audio chunk is bigger than 200ms, push at
|
||||||
# every audio chunk.
|
# every audio chunk.
|
||||||
@@ -518,23 +699,36 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def _create_video_task(self):
|
def _create_video_task(self):
|
||||||
|
"""Create the video processing task if video output is enabled."""
|
||||||
if not self._video_task and self._params.video_out_enabled:
|
if not self._video_task and self._params.video_out_enabled:
|
||||||
self._video_queue = asyncio.Queue()
|
self._video_queue = asyncio.Queue()
|
||||||
self._video_task = self._transport.create_task(self._video_task_handler())
|
self._video_task = self._transport.create_task(self._video_task_handler())
|
||||||
|
|
||||||
async def _cancel_video_task(self):
|
async def _cancel_video_task(self):
|
||||||
|
"""Cancel and cleanup the video processing task."""
|
||||||
# Stop video output task.
|
# Stop video output task.
|
||||||
if self._video_task:
|
if self._video_task:
|
||||||
await self._transport.cancel_task(self._video_task)
|
await self._transport.cancel_task(self._video_task)
|
||||||
self._video_task = None
|
self._video_task = None
|
||||||
|
|
||||||
async def _set_video_image(self, image: OutputImageRawFrame):
|
async def _set_video_image(self, image: OutputImageRawFrame):
|
||||||
|
"""Set a single video image for cycling output.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
image: The image frame to cycle for video output.
|
||||||
|
"""
|
||||||
self._video_images = itertools.cycle([image])
|
self._video_images = itertools.cycle([image])
|
||||||
|
|
||||||
async def _set_video_images(self, images: List[OutputImageRawFrame]):
|
async def _set_video_images(self, images: List[OutputImageRawFrame]):
|
||||||
|
"""Set multiple video images for cycling output.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
images: The list of image frames to cycle for video output.
|
||||||
|
"""
|
||||||
self._video_images = itertools.cycle(images)
|
self._video_images = itertools.cycle(images)
|
||||||
|
|
||||||
async def _video_task_handler(self):
|
async def _video_task_handler(self):
|
||||||
|
"""Main video processing task handler."""
|
||||||
self._video_start_time = None
|
self._video_start_time = None
|
||||||
self._video_frame_index = 0
|
self._video_frame_index = 0
|
||||||
self._video_frame_duration = 1 / self._params.video_out_framerate
|
self._video_frame_duration = 1 / self._params.video_out_framerate
|
||||||
@@ -550,6 +744,7 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
await asyncio.sleep(self._video_frame_duration)
|
await asyncio.sleep(self._video_frame_duration)
|
||||||
|
|
||||||
async def _video_is_live_handler(self):
|
async def _video_is_live_handler(self):
|
||||||
|
"""Handle live video streaming with frame timing."""
|
||||||
image = await self._video_queue.get()
|
image = await self._video_queue.get()
|
||||||
|
|
||||||
# We get the start time as soon as we get the first image.
|
# We get the start time as soon as we get the first image.
|
||||||
@@ -575,6 +770,12 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
self._video_queue.task_done()
|
self._video_queue.task_done()
|
||||||
|
|
||||||
async def _draw_image(self, frame: OutputImageRawFrame):
|
async def _draw_image(self, frame: OutputImageRawFrame):
|
||||||
|
"""Draw/render an image frame with resizing if needed.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The image frame to draw.
|
||||||
|
"""
|
||||||
|
|
||||||
def resize_frame(frame: OutputImageRawFrame) -> OutputImageRawFrame:
|
def resize_frame(frame: OutputImageRawFrame) -> OutputImageRawFrame:
|
||||||
desired_size = (self._params.video_out_width, self._params.video_out_height)
|
desired_size = (self._params.video_out_width, self._params.video_out_height)
|
||||||
|
|
||||||
@@ -601,16 +802,19 @@ class BaseOutputTransport(FrameProcessor):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def _create_clock_task(self):
|
def _create_clock_task(self):
|
||||||
|
"""Create the clock/timing processing task."""
|
||||||
if not self._clock_task:
|
if not self._clock_task:
|
||||||
self._clock_queue = WatchdogPriorityQueue(self._transport.task_manager)
|
self._clock_queue = WatchdogPriorityQueue(self._transport.task_manager)
|
||||||
self._clock_task = self._transport.create_task(self._clock_task_handler())
|
self._clock_task = self._transport.create_task(self._clock_task_handler())
|
||||||
|
|
||||||
async def _cancel_clock_task(self):
|
async def _cancel_clock_task(self):
|
||||||
|
"""Cancel and cleanup the clock processing task."""
|
||||||
if self._clock_task:
|
if self._clock_task:
|
||||||
await self._transport.cancel_task(self._clock_task)
|
await self._transport.cancel_task(self._clock_task)
|
||||||
self._clock_task = None
|
self._clock_task = None
|
||||||
|
|
||||||
async def _clock_task_handler(self):
|
async def _clock_task_handler(self):
|
||||||
|
"""Main clock/timing task handler for timed frame delivery."""
|
||||||
running = True
|
running = True
|
||||||
while running:
|
while running:
|
||||||
timestamp, _, frame = await self._clock_queue.get()
|
timestamp, _, frame = await self._clock_queue.get()
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Base transport classes for Pipecat.
|
||||||
|
|
||||||
|
This module provides the foundation for transport implementations including
|
||||||
|
parameter configuration and abstract base classes for input/output transport
|
||||||
|
functionality.
|
||||||
|
"""
|
||||||
|
|
||||||
from abc import abstractmethod
|
from abc import abstractmethod
|
||||||
from typing import List, Mapping, Optional
|
from typing import List, Mapping, Optional
|
||||||
|
|
||||||
@@ -18,6 +25,45 @@ from pipecat.utils.base_object import BaseObject
|
|||||||
|
|
||||||
|
|
||||||
class TransportParams(BaseModel):
|
class TransportParams(BaseModel):
|
||||||
|
"""Configuration parameters for transport implementations.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
camera_in_enabled: Enable camera input (deprecated, use video_in_enabled).
|
||||||
|
camera_out_enabled: Enable camera output (deprecated, use video_out_enabled).
|
||||||
|
camera_out_is_live: Enable real-time camera output (deprecated).
|
||||||
|
camera_out_width: Camera output width in pixels (deprecated).
|
||||||
|
camera_out_height: Camera output height in pixels (deprecated).
|
||||||
|
camera_out_bitrate: Camera output bitrate in bits per second (deprecated).
|
||||||
|
camera_out_framerate: Camera output frame rate in FPS (deprecated).
|
||||||
|
camera_out_color_format: Camera output color format string (deprecated).
|
||||||
|
audio_out_enabled: Enable audio output streaming.
|
||||||
|
audio_out_sample_rate: Output audio sample rate in Hz.
|
||||||
|
audio_out_channels: Number of output audio channels.
|
||||||
|
audio_out_bitrate: Output audio bitrate in bits per second.
|
||||||
|
audio_out_10ms_chunks: Number of 10ms chunks to buffer for output.
|
||||||
|
audio_out_mixer: Audio mixer instance or destination mapping.
|
||||||
|
audio_out_destinations: List of audio output destination identifiers.
|
||||||
|
audio_in_enabled: Enable audio input streaming.
|
||||||
|
audio_in_sample_rate: Input audio sample rate in Hz.
|
||||||
|
audio_in_channels: Number of input audio channels.
|
||||||
|
audio_in_filter: Audio filter to apply to input audio.
|
||||||
|
audio_in_stream_on_start: Start audio streaming immediately on transport start.
|
||||||
|
audio_in_passthrough: Pass through input audio frames downstream.
|
||||||
|
video_in_enabled: Enable video input streaming.
|
||||||
|
video_out_enabled: Enable video output streaming.
|
||||||
|
video_out_is_live: Enable real-time video output streaming.
|
||||||
|
video_out_width: Video output width in pixels.
|
||||||
|
video_out_height: Video output height in pixels.
|
||||||
|
video_out_bitrate: Video output bitrate in bits per second.
|
||||||
|
video_out_framerate: Video output frame rate in FPS.
|
||||||
|
video_out_color_format: Video output color format string.
|
||||||
|
video_out_destinations: List of video output destination identifiers.
|
||||||
|
vad_enabled: Enable Voice Activity Detection (deprecated).
|
||||||
|
vad_audio_passthrough: Enable VAD audio passthrough (deprecated).
|
||||||
|
vad_analyzer: Voice Activity Detection analyzer instance.
|
||||||
|
turn_analyzer: Turn-taking analyzer instance for conversation management.
|
||||||
|
"""
|
||||||
|
|
||||||
model_config = ConfigDict(arbitrary_types_allowed=True)
|
model_config = ConfigDict(arbitrary_types_allowed=True)
|
||||||
|
|
||||||
camera_in_enabled: bool = False
|
camera_in_enabled: bool = False
|
||||||
@@ -57,6 +103,12 @@ class TransportParams(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class BaseTransport(BaseObject):
|
class BaseTransport(BaseObject):
|
||||||
|
"""Base class for transport implementations.
|
||||||
|
|
||||||
|
Provides the foundation for transport classes that handle media streaming,
|
||||||
|
including input and output frame processors for audio and video data.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
*,
|
*,
|
||||||
@@ -64,14 +116,31 @@ class BaseTransport(BaseObject):
|
|||||||
input_name: Optional[str] = None,
|
input_name: Optional[str] = None,
|
||||||
output_name: Optional[str] = None,
|
output_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the base transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: Optional name for the transport instance.
|
||||||
|
input_name: Optional name for the input processor.
|
||||||
|
output_name: Optional name for the output processor.
|
||||||
|
"""
|
||||||
super().__init__(name=name)
|
super().__init__(name=name)
|
||||||
self._input_name = input_name
|
self._input_name = input_name
|
||||||
self._output_name = output_name
|
self._output_name = output_name
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def input(self) -> FrameProcessor:
|
def input(self) -> FrameProcessor:
|
||||||
|
"""Get the input frame processor for this transport.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The frame processor that handles incoming frames.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def output(self) -> FrameProcessor:
|
def output(self) -> FrameProcessor:
|
||||||
|
"""Get the output frame processor for this transport.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The frame processor that handles outgoing frames.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Local audio transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides a local audio transport that uses PyAudio for real-time
|
||||||
|
audio input and output through the system's default audio devices.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
@@ -27,14 +33,33 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class LocalAudioTransportParams(TransportParams):
|
class LocalAudioTransportParams(TransportParams):
|
||||||
|
"""Configuration parameters for local audio transport.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
input_device_index: PyAudio device index for audio input. If None, uses default.
|
||||||
|
output_device_index: PyAudio device index for audio output. If None, uses default.
|
||||||
|
"""
|
||||||
|
|
||||||
input_device_index: Optional[int] = None
|
input_device_index: Optional[int] = None
|
||||||
output_device_index: Optional[int] = None
|
output_device_index: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
class LocalAudioInputTransport(BaseInputTransport):
|
class LocalAudioInputTransport(BaseInputTransport):
|
||||||
|
"""Local audio input transport using PyAudio.
|
||||||
|
|
||||||
|
Captures audio from the system's audio input device and converts it to
|
||||||
|
InputAudioRawFrame objects for processing in the pipeline.
|
||||||
|
"""
|
||||||
|
|
||||||
_params: LocalAudioTransportParams
|
_params: LocalAudioTransportParams
|
||||||
|
|
||||||
def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams):
|
def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams):
|
||||||
|
"""Initialize the local audio input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
py_audio: PyAudio instance for audio device management.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
"""
|
||||||
super().__init__(params)
|
super().__init__(params)
|
||||||
self._py_audio = py_audio
|
self._py_audio = py_audio
|
||||||
|
|
||||||
@@ -42,6 +67,11 @@ class LocalAudioInputTransport(BaseInputTransport):
|
|||||||
self._sample_rate = 0
|
self._sample_rate = 0
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the audio input stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._in_stream:
|
if self._in_stream:
|
||||||
@@ -64,6 +94,7 @@ class LocalAudioInputTransport(BaseInputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Stop and cleanup the audio input stream."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
if self._in_stream:
|
if self._in_stream:
|
||||||
self._in_stream.stop_stream()
|
self._in_stream.stop_stream()
|
||||||
@@ -71,6 +102,7 @@ class LocalAudioInputTransport(BaseInputTransport):
|
|||||||
self._in_stream = None
|
self._in_stream = None
|
||||||
|
|
||||||
def _audio_in_callback(self, in_data, frame_count, time_info, status):
|
def _audio_in_callback(self, in_data, frame_count, time_info, status):
|
||||||
|
"""Callback function for PyAudio input stream."""
|
||||||
frame = InputAudioRawFrame(
|
frame = InputAudioRawFrame(
|
||||||
audio=in_data,
|
audio=in_data,
|
||||||
sample_rate=self._sample_rate,
|
sample_rate=self._sample_rate,
|
||||||
@@ -83,9 +115,21 @@ class LocalAudioInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class LocalAudioOutputTransport(BaseOutputTransport):
|
class LocalAudioOutputTransport(BaseOutputTransport):
|
||||||
|
"""Local audio output transport using PyAudio.
|
||||||
|
|
||||||
|
Plays audio frames through the system's audio output device by converting
|
||||||
|
OutputAudioRawFrame objects to playable audio data.
|
||||||
|
"""
|
||||||
|
|
||||||
_params: LocalAudioTransportParams
|
_params: LocalAudioTransportParams
|
||||||
|
|
||||||
def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams):
|
def __init__(self, py_audio: pyaudio.PyAudio, params: LocalAudioTransportParams):
|
||||||
|
"""Initialize the local audio output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
py_audio: PyAudio instance for audio device management.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
"""
|
||||||
super().__init__(params)
|
super().__init__(params)
|
||||||
self._py_audio = py_audio
|
self._py_audio = py_audio
|
||||||
|
|
||||||
@@ -97,6 +141,11 @@ class LocalAudioOutputTransport(BaseOutputTransport):
|
|||||||
self._executor = ThreadPoolExecutor(max_workers=1)
|
self._executor = ThreadPoolExecutor(max_workers=1)
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the audio output stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._out_stream:
|
if self._out_stream:
|
||||||
@@ -116,6 +165,7 @@ class LocalAudioOutputTransport(BaseOutputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Stop and cleanup the audio output stream."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
if self._out_stream:
|
if self._out_stream:
|
||||||
self._out_stream.stop_stream()
|
self._out_stream.stop_stream()
|
||||||
@@ -123,6 +173,11 @@ class LocalAudioOutputTransport(BaseOutputTransport):
|
|||||||
self._out_stream = None
|
self._out_stream = None
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the output stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to write to the output device.
|
||||||
|
"""
|
||||||
if self._out_stream:
|
if self._out_stream:
|
||||||
await self.get_event_loop().run_in_executor(
|
await self.get_event_loop().run_in_executor(
|
||||||
self._executor, self._out_stream.write, frame.audio
|
self._executor, self._out_stream.write, frame.audio
|
||||||
@@ -130,7 +185,18 @@ class LocalAudioOutputTransport(BaseOutputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class LocalAudioTransport(BaseTransport):
|
class LocalAudioTransport(BaseTransport):
|
||||||
|
"""Complete local audio transport with input and output capabilities.
|
||||||
|
|
||||||
|
Provides a unified interface for local audio I/O using PyAudio, supporting
|
||||||
|
both audio capture and playback through the system's audio devices.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, params: LocalAudioTransportParams):
|
def __init__(self, params: LocalAudioTransportParams):
|
||||||
|
"""Initialize the local audio transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._params = params
|
self._params = params
|
||||||
self._pyaudio = pyaudio.PyAudio()
|
self._pyaudio = pyaudio.PyAudio()
|
||||||
@@ -143,11 +209,21 @@ class LocalAudioTransport(BaseTransport):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def input(self) -> FrameProcessor:
|
def input(self) -> FrameProcessor:
|
||||||
|
"""Get the input frame processor for this transport.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The audio input transport processor.
|
||||||
|
"""
|
||||||
if not self._input:
|
if not self._input:
|
||||||
self._input = LocalAudioInputTransport(self._pyaudio, self._params)
|
self._input = LocalAudioInputTransport(self._pyaudio, self._params)
|
||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> FrameProcessor:
|
def output(self) -> FrameProcessor:
|
||||||
|
"""Get the output frame processor for this transport.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The audio output transport processor.
|
||||||
|
"""
|
||||||
if not self._output:
|
if not self._output:
|
||||||
self._output = LocalAudioOutputTransport(self._pyaudio, self._params)
|
self._output = LocalAudioOutputTransport(self._pyaudio, self._params)
|
||||||
return self._output
|
return self._output
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Tkinter-based local transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides a local transport using Tkinter for video display and
|
||||||
|
PyAudio for audio I/O, suitable for desktop applications and testing.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import tkinter as tk
|
import tkinter as tk
|
||||||
from concurrent.futures import ThreadPoolExecutor
|
from concurrent.futures import ThreadPoolExecutor
|
||||||
@@ -40,20 +46,44 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class TkTransportParams(TransportParams):
|
class TkTransportParams(TransportParams):
|
||||||
|
"""Configuration parameters for Tkinter transport.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
audio_input_device_index: PyAudio device index for audio input. If None, uses default.
|
||||||
|
audio_output_device_index: PyAudio device index for audio output. If None, uses default.
|
||||||
|
"""
|
||||||
|
|
||||||
audio_input_device_index: Optional[int] = None
|
audio_input_device_index: Optional[int] = None
|
||||||
audio_output_device_index: Optional[int] = None
|
audio_output_device_index: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
class TkInputTransport(BaseInputTransport):
|
class TkInputTransport(BaseInputTransport):
|
||||||
|
"""Tkinter-based audio input transport.
|
||||||
|
|
||||||
|
Captures audio from the system's audio input device using PyAudio and
|
||||||
|
converts it to InputAudioRawFrame objects for pipeline processing.
|
||||||
|
"""
|
||||||
|
|
||||||
_params: TkTransportParams
|
_params: TkTransportParams
|
||||||
|
|
||||||
def __init__(self, py_audio: pyaudio.PyAudio, params: TkTransportParams):
|
def __init__(self, py_audio: pyaudio.PyAudio, params: TkTransportParams):
|
||||||
|
"""Initialize the Tkinter input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
py_audio: PyAudio instance for audio device management.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
"""
|
||||||
super().__init__(params)
|
super().__init__(params)
|
||||||
self._py_audio = py_audio
|
self._py_audio = py_audio
|
||||||
self._in_stream = None
|
self._in_stream = None
|
||||||
self._sample_rate = 0
|
self._sample_rate = 0
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the audio input stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._in_stream:
|
if self._in_stream:
|
||||||
@@ -76,6 +106,7 @@ class TkInputTransport(BaseInputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Stop and cleanup the audio input stream."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
if self._in_stream:
|
if self._in_stream:
|
||||||
self._in_stream.stop_stream()
|
self._in_stream.stop_stream()
|
||||||
@@ -83,6 +114,7 @@ class TkInputTransport(BaseInputTransport):
|
|||||||
self._in_stream = None
|
self._in_stream = None
|
||||||
|
|
||||||
def _audio_in_callback(self, in_data, frame_count, time_info, status):
|
def _audio_in_callback(self, in_data, frame_count, time_info, status):
|
||||||
|
"""Callback function for PyAudio input stream."""
|
||||||
frame = InputAudioRawFrame(
|
frame = InputAudioRawFrame(
|
||||||
audio=in_data,
|
audio=in_data,
|
||||||
sample_rate=self._sample_rate,
|
sample_rate=self._sample_rate,
|
||||||
@@ -95,9 +127,22 @@ class TkInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class TkOutputTransport(BaseOutputTransport):
|
class TkOutputTransport(BaseOutputTransport):
|
||||||
|
"""Tkinter-based audio and video output transport.
|
||||||
|
|
||||||
|
Plays audio through PyAudio and displays video frames in a Tkinter window,
|
||||||
|
providing a complete multimedia output solution for desktop applications.
|
||||||
|
"""
|
||||||
|
|
||||||
_params: TkTransportParams
|
_params: TkTransportParams
|
||||||
|
|
||||||
def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TkTransportParams):
|
def __init__(self, tk_root: tk.Tk, py_audio: pyaudio.PyAudio, params: TkTransportParams):
|
||||||
|
"""Initialize the Tkinter output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
tk_root: The root Tkinter window for video display.
|
||||||
|
py_audio: PyAudio instance for audio device management.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
"""
|
||||||
super().__init__(params)
|
super().__init__(params)
|
||||||
self._py_audio = py_audio
|
self._py_audio = py_audio
|
||||||
self._out_stream = None
|
self._out_stream = None
|
||||||
@@ -115,6 +160,11 @@ class TkOutputTransport(BaseOutputTransport):
|
|||||||
self._image_label.pack()
|
self._image_label.pack()
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the audio output stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._out_stream:
|
if self._out_stream:
|
||||||
@@ -134,6 +184,7 @@ class TkOutputTransport(BaseOutputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Stop and cleanup the audio output stream."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
if self._out_stream:
|
if self._out_stream:
|
||||||
self._out_stream.stop_stream()
|
self._out_stream.stop_stream()
|
||||||
@@ -141,15 +192,26 @@ class TkOutputTransport(BaseOutputTransport):
|
|||||||
self._out_stream = None
|
self._out_stream = None
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the output stream.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to write to the output device.
|
||||||
|
"""
|
||||||
if self._out_stream:
|
if self._out_stream:
|
||||||
await self.get_event_loop().run_in_executor(
|
await self.get_event_loop().run_in_executor(
|
||||||
self._executor, self._out_stream.write, frame.audio
|
self._executor, self._out_stream.write, frame.audio
|
||||||
)
|
)
|
||||||
|
|
||||||
async def write_video_frame(self, frame: OutputImageRawFrame):
|
async def write_video_frame(self, frame: OutputImageRawFrame):
|
||||||
|
"""Write a video frame to the Tkinter display.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The video frame to display in the Tkinter window.
|
||||||
|
"""
|
||||||
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
|
self.get_event_loop().call_soon(self._write_frame_to_tk, frame)
|
||||||
|
|
||||||
def _write_frame_to_tk(self, frame: OutputImageRawFrame):
|
def _write_frame_to_tk(self, frame: OutputImageRawFrame):
|
||||||
|
"""Write frame data to the Tkinter image label."""
|
||||||
width = frame.size[0]
|
width = frame.size[0]
|
||||||
height = frame.size[1]
|
height = frame.size[1]
|
||||||
data = f"P6 {width} {height} 255 ".encode() + frame.image
|
data = f"P6 {width} {height} 255 ".encode() + frame.image
|
||||||
@@ -162,7 +224,19 @@ class TkOutputTransport(BaseOutputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class TkLocalTransport(BaseTransport):
|
class TkLocalTransport(BaseTransport):
|
||||||
|
"""Complete Tkinter-based local transport with audio and video capabilities.
|
||||||
|
|
||||||
|
Provides a unified interface for local multimedia I/O using Tkinter for video
|
||||||
|
display and PyAudio for audio, suitable for desktop applications and testing.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, tk_root: tk.Tk, params: TkTransportParams):
|
def __init__(self, tk_root: tk.Tk, params: TkTransportParams):
|
||||||
|
"""Initialize the Tkinter local transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
tk_root: The root Tkinter window for video display.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._tk_root = tk_root
|
self._tk_root = tk_root
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -176,11 +250,21 @@ class TkLocalTransport(BaseTransport):
|
|||||||
#
|
#
|
||||||
|
|
||||||
def input(self) -> TkInputTransport:
|
def input(self) -> TkInputTransport:
|
||||||
|
"""Get the input frame processor for this transport.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The Tkinter input transport processor.
|
||||||
|
"""
|
||||||
if not self._input:
|
if not self._input:
|
||||||
self._input = TkInputTransport(self._pyaudio, self._params)
|
self._input = TkInputTransport(self._pyaudio, self._params)
|
||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> TkOutputTransport:
|
def output(self) -> TkOutputTransport:
|
||||||
|
"""Get the output frame processor for this transport.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The Tkinter output transport processor.
|
||||||
|
"""
|
||||||
if not self._output:
|
if not self._output:
|
||||||
self._output = TkOutputTransport(self._tk_root, self._pyaudio, self._params)
|
self._output = TkOutputTransport(self._tk_root, self._pyaudio, self._params)
|
||||||
return self._output
|
return self._output
|
||||||
|
|||||||
@@ -4,6 +4,12 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""FastAPI WebSocket transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides WebSocket-based transport for real-time audio/video streaming
|
||||||
|
using FastAPI and WebSocket connections. Supports binary and text serialization
|
||||||
|
with configurable session timeouts and WAV header generation.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import io
|
import io
|
||||||
@@ -45,19 +51,48 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class FastAPIWebsocketParams(TransportParams):
|
class FastAPIWebsocketParams(TransportParams):
|
||||||
|
"""Configuration parameters for FastAPI WebSocket transport.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
add_wav_header: Whether to add WAV headers to audio frames.
|
||||||
|
serializer: Frame serializer for encoding/decoding messages.
|
||||||
|
session_timeout: Session timeout in seconds, None for no timeout.
|
||||||
|
"""
|
||||||
|
|
||||||
add_wav_header: bool = False
|
add_wav_header: bool = False
|
||||||
serializer: Optional[FrameSerializer] = None
|
serializer: Optional[FrameSerializer] = None
|
||||||
session_timeout: Optional[int] = None
|
session_timeout: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
class FastAPIWebsocketCallbacks(BaseModel):
|
class FastAPIWebsocketCallbacks(BaseModel):
|
||||||
|
"""Callback functions for WebSocket events.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
on_client_connected: Called when a client connects to the WebSocket.
|
||||||
|
on_client_disconnected: Called when a client disconnects from the WebSocket.
|
||||||
|
on_session_timeout: Called when a session timeout occurs.
|
||||||
|
"""
|
||||||
|
|
||||||
on_client_connected: Callable[[WebSocket], Awaitable[None]]
|
on_client_connected: Callable[[WebSocket], Awaitable[None]]
|
||||||
on_client_disconnected: Callable[[WebSocket], Awaitable[None]]
|
on_client_disconnected: Callable[[WebSocket], Awaitable[None]]
|
||||||
on_session_timeout: Callable[[WebSocket], Awaitable[None]]
|
on_session_timeout: Callable[[WebSocket], Awaitable[None]]
|
||||||
|
|
||||||
|
|
||||||
class FastAPIWebsocketClient:
|
class FastAPIWebsocketClient:
|
||||||
|
"""WebSocket client wrapper for handling connections and message passing.
|
||||||
|
|
||||||
|
Manages WebSocket state, message sending/receiving, and connection lifecycle
|
||||||
|
with support for both binary and text message types.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, websocket: WebSocket, is_binary: bool, callbacks: FastAPIWebsocketCallbacks):
|
def __init__(self, websocket: WebSocket, is_binary: bool, callbacks: FastAPIWebsocketCallbacks):
|
||||||
|
"""Initialize the WebSocket client.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
websocket: The FastAPI WebSocket connection.
|
||||||
|
is_binary: Whether to use binary message format.
|
||||||
|
callbacks: Event callback functions.
|
||||||
|
"""
|
||||||
self._websocket = websocket
|
self._websocket = websocket
|
||||||
self._closing = False
|
self._closing = False
|
||||||
self._is_binary = is_binary
|
self._is_binary = is_binary
|
||||||
@@ -65,12 +100,27 @@ class FastAPIWebsocketClient:
|
|||||||
self._leave_counter = 0
|
self._leave_counter = 0
|
||||||
|
|
||||||
async def setup(self, _: StartFrame):
|
async def setup(self, _: StartFrame):
|
||||||
|
"""Set up the WebSocket client.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
_: The start frame (unused).
|
||||||
|
"""
|
||||||
self._leave_counter += 1
|
self._leave_counter += 1
|
||||||
|
|
||||||
def receive(self) -> typing.AsyncIterator[bytes | str]:
|
def receive(self) -> typing.AsyncIterator[bytes | str]:
|
||||||
|
"""Get an async iterator for receiving WebSocket messages.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
An async iterator yielding bytes or strings based on message type.
|
||||||
|
"""
|
||||||
return self._websocket.iter_bytes() if self._is_binary else self._websocket.iter_text()
|
return self._websocket.iter_bytes() if self._is_binary else self._websocket.iter_text()
|
||||||
|
|
||||||
async def send(self, data: str | bytes):
|
async def send(self, data: str | bytes):
|
||||||
|
"""Send data through the WebSocket connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: The data to send (string or bytes).
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
if self._can_send():
|
if self._can_send():
|
||||||
if self._is_binary:
|
if self._is_binary:
|
||||||
@@ -89,6 +139,7 @@ class FastAPIWebsocketClient:
|
|||||||
await self.trigger_client_disconnected()
|
await self.trigger_client_disconnected()
|
||||||
|
|
||||||
async def disconnect(self):
|
async def disconnect(self):
|
||||||
|
"""Disconnect the WebSocket client."""
|
||||||
self._leave_counter -= 1
|
self._leave_counter -= 1
|
||||||
if self._leave_counter > 0:
|
if self._leave_counter > 0:
|
||||||
return
|
return
|
||||||
@@ -99,27 +150,47 @@ class FastAPIWebsocketClient:
|
|||||||
await self.trigger_client_disconnected()
|
await self.trigger_client_disconnected()
|
||||||
|
|
||||||
async def trigger_client_disconnected(self):
|
async def trigger_client_disconnected(self):
|
||||||
|
"""Trigger the client disconnected callback."""
|
||||||
await self._callbacks.on_client_disconnected(self._websocket)
|
await self._callbacks.on_client_disconnected(self._websocket)
|
||||||
|
|
||||||
async def trigger_client_connected(self):
|
async def trigger_client_connected(self):
|
||||||
|
"""Trigger the client connected callback."""
|
||||||
await self._callbacks.on_client_connected(self._websocket)
|
await self._callbacks.on_client_connected(self._websocket)
|
||||||
|
|
||||||
async def trigger_client_timeout(self):
|
async def trigger_client_timeout(self):
|
||||||
|
"""Trigger the client timeout callback."""
|
||||||
await self._callbacks.on_session_timeout(self._websocket)
|
await self._callbacks.on_session_timeout(self._websocket)
|
||||||
|
|
||||||
def _can_send(self):
|
def _can_send(self):
|
||||||
|
"""Check if data can be sent through the WebSocket."""
|
||||||
return self.is_connected and not self.is_closing
|
return self.is_connected and not self.is_closing
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_connected(self) -> bool:
|
def is_connected(self) -> bool:
|
||||||
|
"""Check if the WebSocket is currently connected.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the WebSocket is in connected state.
|
||||||
|
"""
|
||||||
return self._websocket.client_state == WebSocketState.CONNECTED
|
return self._websocket.client_state == WebSocketState.CONNECTED
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_closing(self) -> bool:
|
def is_closing(self) -> bool:
|
||||||
|
"""Check if the WebSocket is currently closing.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the WebSocket is in the process of closing.
|
||||||
|
"""
|
||||||
return self._closing
|
return self._closing
|
||||||
|
|
||||||
|
|
||||||
class FastAPIWebsocketInputTransport(BaseInputTransport):
|
class FastAPIWebsocketInputTransport(BaseInputTransport):
|
||||||
|
"""Input transport for FastAPI WebSocket connections.
|
||||||
|
|
||||||
|
Handles incoming WebSocket messages, deserializes frames, and manages
|
||||||
|
connection monitoring with optional session timeouts.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: BaseTransport,
|
transport: BaseTransport,
|
||||||
@@ -127,6 +198,14 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
|
|||||||
params: FastAPIWebsocketParams,
|
params: FastAPIWebsocketParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
client: The WebSocket client wrapper.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._client = client
|
self._client = client
|
||||||
@@ -138,6 +217,11 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the input transport and begin message processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -156,6 +240,7 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def _stop_tasks(self):
|
async def _stop_tasks(self):
|
||||||
|
"""Stop all running tasks."""
|
||||||
if self._monitor_websocket_task:
|
if self._monitor_websocket_task:
|
||||||
await self.cancel_task(self._monitor_websocket_task)
|
await self.cancel_task(self._monitor_websocket_task)
|
||||||
self._monitor_websocket_task = None
|
self._monitor_websocket_task = None
|
||||||
@@ -164,20 +249,32 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
|
|||||||
self._receive_task = None
|
self._receive_task = None
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the input transport and cleanup resources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._stop_tasks()
|
await self._stop_tasks()
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the input transport and stop all processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._stop_tasks()
|
await self._stop_tasks()
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Clean up transport resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def _receive_messages(self):
|
async def _receive_messages(self):
|
||||||
|
"""Main message receiving loop for WebSocket messages."""
|
||||||
try:
|
try:
|
||||||
async for message in WatchdogAsyncIterator(
|
async for message in WatchdogAsyncIterator(
|
||||||
self._client.receive(), manager=self.task_manager
|
self._client.receive(), manager=self.task_manager
|
||||||
@@ -206,6 +303,12 @@ class FastAPIWebsocketInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
||||||
|
"""Output transport for FastAPI WebSocket connections.
|
||||||
|
|
||||||
|
Handles outgoing frame serialization, audio streaming with timing simulation,
|
||||||
|
and WebSocket message transmission with optional WAV header generation.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: BaseTransport,
|
transport: BaseTransport,
|
||||||
@@ -213,6 +316,14 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
|||||||
params: FastAPIWebsocketParams,
|
params: FastAPIWebsocketParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
client: The WebSocket client wrapper.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
|
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
@@ -231,6 +342,11 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the output transport and initialize timing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -245,20 +361,37 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the output transport and cleanup resources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._write_frame(frame)
|
await self._write_frame(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the output transport and stop all processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._write_frame(frame)
|
await self._write_frame(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Clean up transport resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process outgoing frames with special handling for interruptions.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow in the pipeline.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
if isinstance(frame, StartInterruptionFrame):
|
if isinstance(frame, StartInterruptionFrame):
|
||||||
@@ -266,9 +399,19 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
|||||||
self._next_send_time = 0
|
self._next_send_time = 0
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a transport message frame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The transport message frame to send.
|
||||||
|
"""
|
||||||
await self._write_frame(frame)
|
await self._write_frame(frame)
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the WebSocket with timing simulation.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output audio frame to write.
|
||||||
|
"""
|
||||||
if self._client.is_closing:
|
if self._client.is_closing:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -303,6 +446,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
|||||||
await self._write_audio_sleep()
|
await self._write_audio_sleep()
|
||||||
|
|
||||||
async def _write_frame(self, frame: Frame):
|
async def _write_frame(self, frame: Frame):
|
||||||
|
"""Serialize and send a frame through the WebSocket."""
|
||||||
if not self._params.serializer:
|
if not self._params.serializer:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -314,6 +458,7 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
|||||||
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
|
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
|
||||||
|
|
||||||
async def _write_audio_sleep(self):
|
async def _write_audio_sleep(self):
|
||||||
|
"""Simulate audio playback timing with appropriate delays."""
|
||||||
# Simulate a clock.
|
# Simulate a clock.
|
||||||
current_time = time.monotonic()
|
current_time = time.monotonic()
|
||||||
sleep_duration = max(0, self._next_send_time - current_time)
|
sleep_duration = max(0, self._next_send_time - current_time)
|
||||||
@@ -325,6 +470,12 @@ class FastAPIWebsocketOutputTransport(BaseOutputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class FastAPIWebsocketTransport(BaseTransport):
|
class FastAPIWebsocketTransport(BaseTransport):
|
||||||
|
"""FastAPI WebSocket transport for real-time audio/video streaming.
|
||||||
|
|
||||||
|
Provides bidirectional WebSocket communication with frame serialization,
|
||||||
|
session management, and event handling for client connections and timeouts.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
websocket: WebSocket,
|
websocket: WebSocket,
|
||||||
@@ -332,6 +483,14 @@ class FastAPIWebsocketTransport(BaseTransport):
|
|||||||
input_name: Optional[str] = None,
|
input_name: Optional[str] = None,
|
||||||
output_name: Optional[str] = None,
|
output_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the FastAPI WebSocket transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
websocket: The FastAPI WebSocket connection.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
input_name: Optional name for the input processor.
|
||||||
|
output_name: Optional name for the output processor.
|
||||||
|
"""
|
||||||
super().__init__(input_name=input_name, output_name=output_name)
|
super().__init__(input_name=input_name, output_name=output_name)
|
||||||
|
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -361,16 +520,29 @@ class FastAPIWebsocketTransport(BaseTransport):
|
|||||||
self._register_event_handler("on_session_timeout")
|
self._register_event_handler("on_session_timeout")
|
||||||
|
|
||||||
def input(self) -> FastAPIWebsocketInputTransport:
|
def input(self) -> FastAPIWebsocketInputTransport:
|
||||||
|
"""Get the input transport processor.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The WebSocket input transport instance.
|
||||||
|
"""
|
||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> FastAPIWebsocketOutputTransport:
|
def output(self) -> FastAPIWebsocketOutputTransport:
|
||||||
|
"""Get the output transport processor.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The WebSocket output transport instance.
|
||||||
|
"""
|
||||||
return self._output
|
return self._output
|
||||||
|
|
||||||
async def _on_client_connected(self, websocket):
|
async def _on_client_connected(self, websocket):
|
||||||
|
"""Handle client connected event."""
|
||||||
await self._call_event_handler("on_client_connected", websocket)
|
await self._call_event_handler("on_client_connected", websocket)
|
||||||
|
|
||||||
async def _on_client_disconnected(self, websocket):
|
async def _on_client_disconnected(self, websocket):
|
||||||
|
"""Handle client disconnected event."""
|
||||||
await self._call_event_handler("on_client_disconnected", websocket)
|
await self._call_event_handler("on_client_disconnected", websocket)
|
||||||
|
|
||||||
async def _on_session_timeout(self, websocket):
|
async def _on_session_timeout(self, websocket):
|
||||||
|
"""Handle session timeout event."""
|
||||||
await self._call_event_handler("on_session_timeout", websocket)
|
await self._call_event_handler("on_session_timeout", websocket)
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Small WebRTC transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides a WebRTC transport implementation using aiortc for
|
||||||
|
real-time audio and video communication. It supports bidirectional media
|
||||||
|
streaming, application messaging, and client connection management.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import fractions
|
import fractions
|
||||||
import time
|
import time
|
||||||
@@ -47,13 +54,32 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class SmallWebRTCCallbacks(BaseModel):
|
class SmallWebRTCCallbacks(BaseModel):
|
||||||
|
"""Callback handlers for SmallWebRTC events.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
on_app_message: Called when an application message is received.
|
||||||
|
on_client_connected: Called when a client establishes connection.
|
||||||
|
on_client_disconnected: Called when a client disconnects.
|
||||||
|
"""
|
||||||
|
|
||||||
on_app_message: Callable[[Any], Awaitable[None]]
|
on_app_message: Callable[[Any], Awaitable[None]]
|
||||||
on_client_connected: Callable[[SmallWebRTCConnection], Awaitable[None]]
|
on_client_connected: Callable[[SmallWebRTCConnection], Awaitable[None]]
|
||||||
on_client_disconnected: Callable[[SmallWebRTCConnection], Awaitable[None]]
|
on_client_disconnected: Callable[[SmallWebRTCConnection], Awaitable[None]]
|
||||||
|
|
||||||
|
|
||||||
class RawAudioTrack(AudioStreamTrack):
|
class RawAudioTrack(AudioStreamTrack):
|
||||||
|
"""Custom audio stream track for WebRTC output.
|
||||||
|
|
||||||
|
Handles audio frame generation and timing for WebRTC transmission,
|
||||||
|
supporting queued audio data with proper synchronization.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, sample_rate):
|
def __init__(self, sample_rate):
|
||||||
|
"""Initialize the raw audio track.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sample_rate: The audio sample rate in Hz.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._sample_rate = sample_rate
|
self._sample_rate = sample_rate
|
||||||
self._samples_per_10ms = sample_rate * 10 // 1000
|
self._samples_per_10ms = sample_rate * 10 // 1000
|
||||||
@@ -64,7 +90,17 @@ class RawAudioTrack(AudioStreamTrack):
|
|||||||
self._chunk_queue = deque()
|
self._chunk_queue = deque()
|
||||||
|
|
||||||
def add_audio_bytes(self, audio_bytes: bytes):
|
def add_audio_bytes(self, audio_bytes: bytes):
|
||||||
"""Adds bytes to the audio buffer and returns a Future that completes when the data is processed."""
|
"""Add audio bytes to the buffer for transmission.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio_bytes: Raw audio data to queue for transmission.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A Future that completes when the data is processed.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ValueError: If audio bytes are not a multiple of 10ms size.
|
||||||
|
"""
|
||||||
if len(audio_bytes) % self._bytes_per_10ms != 0:
|
if len(audio_bytes) % self._bytes_per_10ms != 0:
|
||||||
raise ValueError("Audio bytes must be a multiple of 10ms size.")
|
raise ValueError("Audio bytes must be a multiple of 10ms size.")
|
||||||
future = asyncio.get_running_loop().create_future()
|
future = asyncio.get_running_loop().create_future()
|
||||||
@@ -79,7 +115,11 @@ class RawAudioTrack(AudioStreamTrack):
|
|||||||
return future
|
return future
|
||||||
|
|
||||||
async def recv(self):
|
async def recv(self):
|
||||||
"""Returns the next audio frame, generating silence if needed."""
|
"""Return the next audio frame for WebRTC transmission.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
An AudioFrame containing the next audio data or silence.
|
||||||
|
"""
|
||||||
# Compute required wait time for synchronization
|
# Compute required wait time for synchronization
|
||||||
if self._timestamp > 0:
|
if self._timestamp > 0:
|
||||||
wait = self._start + (self._timestamp / self._sample_rate) - time.time()
|
wait = self._start + (self._timestamp / self._sample_rate) - time.time()
|
||||||
@@ -106,18 +146,37 @@ class RawAudioTrack(AudioStreamTrack):
|
|||||||
|
|
||||||
|
|
||||||
class RawVideoTrack(VideoStreamTrack):
|
class RawVideoTrack(VideoStreamTrack):
|
||||||
|
"""Custom video stream track for WebRTC output.
|
||||||
|
|
||||||
|
Handles video frame queuing and conversion for WebRTC transmission.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, width, height):
|
def __init__(self, width, height):
|
||||||
|
"""Initialize the raw video track.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
width: Video frame width in pixels.
|
||||||
|
height: Video frame height in pixels.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self._width = width
|
self._width = width
|
||||||
self._height = height
|
self._height = height
|
||||||
self._video_buffer = asyncio.Queue()
|
self._video_buffer = asyncio.Queue()
|
||||||
|
|
||||||
def add_video_frame(self, frame):
|
def add_video_frame(self, frame):
|
||||||
"""Adds a raw video frame to the buffer."""
|
"""Add a video frame to the transmission buffer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The video frame to queue for transmission.
|
||||||
|
"""
|
||||||
self._video_buffer.put_nowait(frame)
|
self._video_buffer.put_nowait(frame)
|
||||||
|
|
||||||
async def recv(self):
|
async def recv(self):
|
||||||
"""Returns the next video frame, waiting if the buffer is empty."""
|
"""Return the next video frame for WebRTC transmission.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
A VideoFrame ready for WebRTC transmission.
|
||||||
|
"""
|
||||||
raw_frame = await self._video_buffer.get()
|
raw_frame = await self._video_buffer.get()
|
||||||
|
|
||||||
# Convert bytes to NumPy array
|
# Convert bytes to NumPy array
|
||||||
@@ -134,6 +193,12 @@ class RawVideoTrack(VideoStreamTrack):
|
|||||||
|
|
||||||
|
|
||||||
class SmallWebRTCClient:
|
class SmallWebRTCClient:
|
||||||
|
"""WebRTC client implementation for handling connections and media streams.
|
||||||
|
|
||||||
|
Manages WebRTC peer connections, audio/video streaming, and application
|
||||||
|
messaging through the SmallWebRTCConnection interface.
|
||||||
|
"""
|
||||||
|
|
||||||
FORMAT_CONVERSIONS = {
|
FORMAT_CONVERSIONS = {
|
||||||
"yuv420p": cv2.COLOR_YUV2RGB_I420,
|
"yuv420p": cv2.COLOR_YUV2RGB_I420,
|
||||||
"yuvj420p": cv2.COLOR_YUV2RGB_I420, # OpenCV treats both the same
|
"yuvj420p": cv2.COLOR_YUV2RGB_I420, # OpenCV treats both the same
|
||||||
@@ -142,6 +207,12 @@ class SmallWebRTCClient:
|
|||||||
}
|
}
|
||||||
|
|
||||||
def __init__(self, webrtc_connection: SmallWebRTCConnection, callbacks: SmallWebRTCCallbacks):
|
def __init__(self, webrtc_connection: SmallWebRTCConnection, callbacks: SmallWebRTCCallbacks):
|
||||||
|
"""Initialize the WebRTC client.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
webrtc_connection: The underlying WebRTC connection handler.
|
||||||
|
callbacks: Event callbacks for connection and message handling.
|
||||||
|
"""
|
||||||
self._webrtc_connection = webrtc_connection
|
self._webrtc_connection = webrtc_connection
|
||||||
self._closing = False
|
self._closing = False
|
||||||
self._callbacks = callbacks
|
self._callbacks = callbacks
|
||||||
@@ -180,14 +251,14 @@ class SmallWebRTCClient:
|
|||||||
await self._handle_app_message(message)
|
await self._handle_app_message(message)
|
||||||
|
|
||||||
def _convert_frame(self, frame_array: np.ndarray, format_name: str) -> np.ndarray:
|
def _convert_frame(self, frame_array: np.ndarray, format_name: str) -> np.ndarray:
|
||||||
"""Convert a given frame to RGB format based on the input format.
|
"""Convert a video frame to RGB format based on the input format.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
frame_array (np.ndarray): The input frame.
|
frame_array: The input frame as a NumPy array.
|
||||||
format_name (str): The format of the input frame.
|
format_name: The format of the input frame.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
np.ndarray: The converted RGB frame.
|
The converted RGB frame as a NumPy array.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
ValueError: If the format is unsupported.
|
ValueError: If the format is unsupported.
|
||||||
@@ -203,8 +274,13 @@ class SmallWebRTCClient:
|
|||||||
return cv2.cvtColor(frame_array, conversion_code)
|
return cv2.cvtColor(frame_array, conversion_code)
|
||||||
|
|
||||||
async def read_video_frame(self):
|
async def read_video_frame(self):
|
||||||
"""Reads a video frame from the given MediaStreamTrack, converts it to RGB,
|
"""Read video frames from the WebRTC connection.
|
||||||
|
|
||||||
|
Reads a video frame from the given MediaStreamTrack, converts it to RGB,
|
||||||
and creates an InputImageRawFrame.
|
and creates an InputImageRawFrame.
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
UserImageRawFrame objects containing video data from the peer.
|
||||||
"""
|
"""
|
||||||
while True:
|
while True:
|
||||||
if self._video_input_track is None:
|
if self._video_input_track is None:
|
||||||
@@ -242,7 +318,13 @@ class SmallWebRTCClient:
|
|||||||
yield image_frame
|
yield image_frame
|
||||||
|
|
||||||
async def read_audio_frame(self):
|
async def read_audio_frame(self):
|
||||||
"""Reads 20ms of audio from the given MediaStreamTrack and creates an InputAudioRawFrame."""
|
"""Read audio frames from the WebRTC connection.
|
||||||
|
|
||||||
|
Reads 20ms of audio from the given MediaStreamTrack and creates an InputAudioRawFrame.
|
||||||
|
|
||||||
|
Yields:
|
||||||
|
InputAudioRawFrame objects containing audio data from the peer.
|
||||||
|
"""
|
||||||
while True:
|
while True:
|
||||||
if self._audio_input_track is None:
|
if self._audio_input_track is None:
|
||||||
await asyncio.sleep(0.01)
|
await asyncio.sleep(0.01)
|
||||||
@@ -285,20 +367,37 @@ class SmallWebRTCClient:
|
|||||||
yield audio_frame
|
yield audio_frame
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to transmit.
|
||||||
|
"""
|
||||||
if self._can_send() and self._audio_output_track:
|
if self._can_send() and self._audio_output_track:
|
||||||
await self._audio_output_track.add_audio_bytes(frame.audio)
|
await self._audio_output_track.add_audio_bytes(frame.audio)
|
||||||
|
|
||||||
async def write_video_frame(self, frame: OutputImageRawFrame):
|
async def write_video_frame(self, frame: OutputImageRawFrame):
|
||||||
|
"""Write a video frame to the WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The video frame to transmit.
|
||||||
|
"""
|
||||||
if self._can_send() and self._video_output_track:
|
if self._can_send() and self._video_output_track:
|
||||||
self._video_output_track.add_video_frame(frame)
|
self._video_output_track.add_video_frame(frame)
|
||||||
|
|
||||||
async def setup(self, _params: TransportParams, frame):
|
async def setup(self, _params: TransportParams, frame):
|
||||||
|
"""Set up the client with transport parameters.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
_params: Transport configuration parameters.
|
||||||
|
frame: The initialization frame containing setup data.
|
||||||
|
"""
|
||||||
self._audio_in_channels = _params.audio_in_channels
|
self._audio_in_channels = _params.audio_in_channels
|
||||||
self._in_sample_rate = _params.audio_in_sample_rate or frame.audio_in_sample_rate
|
self._in_sample_rate = _params.audio_in_sample_rate or frame.audio_in_sample_rate
|
||||||
self._out_sample_rate = _params.audio_out_sample_rate or frame.audio_out_sample_rate
|
self._out_sample_rate = _params.audio_out_sample_rate or frame.audio_out_sample_rate
|
||||||
self._params = _params
|
self._params = _params
|
||||||
|
|
||||||
async def connect(self):
|
async def connect(self):
|
||||||
|
"""Establish the WebRTC connection."""
|
||||||
if self._webrtc_connection.is_connected():
|
if self._webrtc_connection.is_connected():
|
||||||
# already initialized
|
# already initialized
|
||||||
return
|
return
|
||||||
@@ -307,6 +406,7 @@ class SmallWebRTCClient:
|
|||||||
await self._webrtc_connection.connect()
|
await self._webrtc_connection.connect()
|
||||||
|
|
||||||
async def disconnect(self):
|
async def disconnect(self):
|
||||||
|
"""Disconnect from the WebRTC peer."""
|
||||||
if self.is_connected and not self.is_closing:
|
if self.is_connected and not self.is_closing:
|
||||||
logger.info(f"Disconnecting to Small WebRTC")
|
logger.info(f"Disconnecting to Small WebRTC")
|
||||||
self._closing = True
|
self._closing = True
|
||||||
@@ -314,10 +414,16 @@ class SmallWebRTCClient:
|
|||||||
await self._handle_peer_disconnected()
|
await self._handle_peer_disconnected()
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send an application message through the WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The message frame to send.
|
||||||
|
"""
|
||||||
if self._can_send():
|
if self._can_send():
|
||||||
self._webrtc_connection.send_app_message(frame.message)
|
self._webrtc_connection.send_app_message(frame.message)
|
||||||
|
|
||||||
async def _handle_client_connected(self):
|
async def _handle_client_connected(self):
|
||||||
|
"""Handle client connection establishment."""
|
||||||
# There is nothing to do here yet, the pipeline is still not ready
|
# There is nothing to do here yet, the pipeline is still not ready
|
||||||
if not self._params:
|
if not self._params:
|
||||||
return
|
return
|
||||||
@@ -337,12 +443,14 @@ class SmallWebRTCClient:
|
|||||||
await self._callbacks.on_client_connected(self._webrtc_connection)
|
await self._callbacks.on_client_connected(self._webrtc_connection)
|
||||||
|
|
||||||
async def _handle_peer_disconnected(self):
|
async def _handle_peer_disconnected(self):
|
||||||
|
"""Handle peer disconnection cleanup."""
|
||||||
self._audio_input_track = None
|
self._audio_input_track = None
|
||||||
self._video_input_track = None
|
self._video_input_track = None
|
||||||
self._audio_output_track = None
|
self._audio_output_track = None
|
||||||
self._video_output_track = None
|
self._video_output_track = None
|
||||||
|
|
||||||
async def _handle_client_closed(self):
|
async def _handle_client_closed(self):
|
||||||
|
"""Handle client connection closure."""
|
||||||
self._audio_input_track = None
|
self._audio_input_track = None
|
||||||
self._video_input_track = None
|
self._video_input_track = None
|
||||||
self._audio_output_track = None
|
self._audio_output_track = None
|
||||||
@@ -350,27 +458,52 @@ class SmallWebRTCClient:
|
|||||||
await self._callbacks.on_client_disconnected(self._webrtc_connection)
|
await self._callbacks.on_client_disconnected(self._webrtc_connection)
|
||||||
|
|
||||||
async def _handle_app_message(self, message: Any):
|
async def _handle_app_message(self, message: Any):
|
||||||
|
"""Handle incoming application messages."""
|
||||||
await self._callbacks.on_app_message(message)
|
await self._callbacks.on_app_message(message)
|
||||||
|
|
||||||
def _can_send(self):
|
def _can_send(self):
|
||||||
|
"""Check if the connection is ready for sending data."""
|
||||||
return self.is_connected and not self.is_closing
|
return self.is_connected and not self.is_closing
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_connected(self) -> bool:
|
def is_connected(self) -> bool:
|
||||||
|
"""Check if the WebRTC connection is established.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if connected to the peer.
|
||||||
|
"""
|
||||||
return self._webrtc_connection.is_connected()
|
return self._webrtc_connection.is_connected()
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def is_closing(self) -> bool:
|
def is_closing(self) -> bool:
|
||||||
|
"""Check if the connection is in the process of closing.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the connection is closing.
|
||||||
|
"""
|
||||||
return self._closing
|
return self._closing
|
||||||
|
|
||||||
|
|
||||||
class SmallWebRTCInputTransport(BaseInputTransport):
|
class SmallWebRTCInputTransport(BaseInputTransport):
|
||||||
|
"""Input transport implementation for SmallWebRTC.
|
||||||
|
|
||||||
|
Handles incoming audio and video streams from WebRTC peers,
|
||||||
|
including user image requests and application message handling.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
client: SmallWebRTCClient,
|
client: SmallWebRTCClient,
|
||||||
params: TransportParams,
|
params: TransportParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebRTC input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: The WebRTC client instance.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
self._client = client
|
self._client = client
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -382,12 +515,23 @@ class SmallWebRTCInputTransport(BaseInputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process incoming frames including user image requests.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow in the pipeline.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
if isinstance(frame, UserImageRequestFrame):
|
if isinstance(frame, UserImageRequestFrame):
|
||||||
await self.request_participant_image(frame)
|
await self.request_participant_image(frame)
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the input transport and establish WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -404,6 +548,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def _stop_tasks(self):
|
async def _stop_tasks(self):
|
||||||
|
"""Stop all background tasks."""
|
||||||
if self._receive_audio_task:
|
if self._receive_audio_task:
|
||||||
await self.cancel_task(self._receive_audio_task)
|
await self.cancel_task(self._receive_audio_task)
|
||||||
self._receive_audio_task = None
|
self._receive_audio_task = None
|
||||||
@@ -412,16 +557,27 @@ class SmallWebRTCInputTransport(BaseInputTransport):
|
|||||||
self._receive_video_task = None
|
self._receive_video_task = None
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the input transport and disconnect from WebRTC.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._stop_tasks()
|
await self._stop_tasks()
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the input transport and disconnect immediately.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._stop_tasks()
|
await self._stop_tasks()
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def _receive_audio(self):
|
async def _receive_audio(self):
|
||||||
|
"""Background task for receiving audio frames from WebRTC."""
|
||||||
try:
|
try:
|
||||||
audio_iterator = self._client.read_audio_frame()
|
audio_iterator = self._client.read_audio_frame()
|
||||||
async for audio_frame in WatchdogAsyncIterator(
|
async for audio_frame in WatchdogAsyncIterator(
|
||||||
@@ -434,6 +590,7 @@ class SmallWebRTCInputTransport(BaseInputTransport):
|
|||||||
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
|
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
|
||||||
|
|
||||||
async def _receive_video(self):
|
async def _receive_video(self):
|
||||||
|
"""Background task for receiving video frames from WebRTC."""
|
||||||
try:
|
try:
|
||||||
video_iterator = self._client.read_video_frame()
|
video_iterator = self._client.read_video_frame()
|
||||||
async for video_frame in WatchdogAsyncIterator(
|
async for video_frame in WatchdogAsyncIterator(
|
||||||
@@ -462,16 +619,24 @@ class SmallWebRTCInputTransport(BaseInputTransport):
|
|||||||
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
|
logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})")
|
||||||
|
|
||||||
async def push_app_message(self, message: Any):
|
async def push_app_message(self, message: Any):
|
||||||
|
"""Push an application message into the pipeline.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
message: The application message to process.
|
||||||
|
"""
|
||||||
logger.debug(f"Received app message inside SmallWebRTCInputTransport {message}")
|
logger.debug(f"Received app message inside SmallWebRTCInputTransport {message}")
|
||||||
frame = TransportMessageUrgentFrame(message=message)
|
frame = TransportMessageUrgentFrame(message=message)
|
||||||
await self.push_frame(frame)
|
await self.push_frame(frame)
|
||||||
|
|
||||||
# Add this method similar to DailyInputTransport.request_participant_image
|
# Add this method similar to DailyInputTransport.request_participant_image
|
||||||
async def request_participant_image(self, frame: UserImageRequestFrame):
|
async def request_participant_image(self, frame: UserImageRequestFrame):
|
||||||
"""Requests an image frame from the participant's video stream.
|
"""Request an image frame from the participant's video stream.
|
||||||
|
|
||||||
When a UserImageRequestFrame is received, this method will store the request
|
When a UserImageRequestFrame is received, this method will store the request
|
||||||
and the next video frame received will be converted to a UserImageRawFrame.
|
and the next video frame received will be converted to a UserImageRawFrame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The user image request frame.
|
||||||
"""
|
"""
|
||||||
logger.debug(f"Requesting image from participant: {frame.user_id}")
|
logger.debug(f"Requesting image from participant: {frame.user_id}")
|
||||||
|
|
||||||
@@ -486,12 +651,25 @@ class SmallWebRTCInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class SmallWebRTCOutputTransport(BaseOutputTransport):
|
class SmallWebRTCOutputTransport(BaseOutputTransport):
|
||||||
|
"""Output transport implementation for SmallWebRTC.
|
||||||
|
|
||||||
|
Handles outgoing audio and video streams to WebRTC peers,
|
||||||
|
including transport message sending.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
client: SmallWebRTCClient,
|
client: SmallWebRTCClient,
|
||||||
params: TransportParams,
|
params: TransportParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebRTC output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: The WebRTC client instance.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
self._client = client
|
self._client = client
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -500,6 +678,11 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the output transport and establish WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -512,24 +695,55 @@ class SmallWebRTCOutputTransport(BaseOutputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the output transport and disconnect from WebRTC.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the output transport and disconnect immediately.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a transport message through the WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The transport message frame to send.
|
||||||
|
"""
|
||||||
await self._client.send_message(frame)
|
await self._client.send_message(frame)
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output audio frame to transmit.
|
||||||
|
"""
|
||||||
await self._client.write_audio_frame(frame)
|
await self._client.write_audio_frame(frame)
|
||||||
|
|
||||||
async def write_video_frame(self, frame: OutputImageRawFrame):
|
async def write_video_frame(self, frame: OutputImageRawFrame):
|
||||||
|
"""Write a video frame to the WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output video frame to transmit.
|
||||||
|
"""
|
||||||
await self._client.write_video_frame(frame)
|
await self._client.write_video_frame(frame)
|
||||||
|
|
||||||
|
|
||||||
class SmallWebRTCTransport(BaseTransport):
|
class SmallWebRTCTransport(BaseTransport):
|
||||||
|
"""WebRTC transport implementation for real-time communication.
|
||||||
|
|
||||||
|
Provides bidirectional audio and video streaming over WebRTC connections
|
||||||
|
with support for application messaging and connection event handling.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
webrtc_connection: SmallWebRTCConnection,
|
webrtc_connection: SmallWebRTCConnection,
|
||||||
@@ -537,6 +751,14 @@ class SmallWebRTCTransport(BaseTransport):
|
|||||||
input_name: Optional[str] = None,
|
input_name: Optional[str] = None,
|
||||||
output_name: Optional[str] = None,
|
output_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebRTC transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
webrtc_connection: The underlying WebRTC connection handler.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
input_name: Optional name for the input processor.
|
||||||
|
output_name: Optional name for the output processor.
|
||||||
|
"""
|
||||||
super().__init__(input_name=input_name, output_name=output_name)
|
super().__init__(input_name=input_name, output_name=output_name)
|
||||||
self._params = params
|
self._params = params
|
||||||
|
|
||||||
@@ -558,6 +780,11 @@ class SmallWebRTCTransport(BaseTransport):
|
|||||||
self._register_event_handler("on_client_disconnected")
|
self._register_event_handler("on_client_disconnected")
|
||||||
|
|
||||||
def input(self) -> SmallWebRTCInputTransport:
|
def input(self) -> SmallWebRTCInputTransport:
|
||||||
|
"""Get the input transport processor.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The input transport for handling incoming media streams.
|
||||||
|
"""
|
||||||
if not self._input:
|
if not self._input:
|
||||||
self._input = SmallWebRTCInputTransport(
|
self._input = SmallWebRTCInputTransport(
|
||||||
self._client, self._params, name=self._input_name
|
self._client, self._params, name=self._input_name
|
||||||
@@ -565,6 +792,11 @@ class SmallWebRTCTransport(BaseTransport):
|
|||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> SmallWebRTCOutputTransport:
|
def output(self) -> SmallWebRTCOutputTransport:
|
||||||
|
"""Get the output transport processor.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The output transport for handling outgoing media streams.
|
||||||
|
"""
|
||||||
if not self._output:
|
if not self._output:
|
||||||
self._output = SmallWebRTCOutputTransport(
|
self._output = SmallWebRTCOutputTransport(
|
||||||
self._client, self._params, name=self._input_name
|
self._client, self._params, name=self._input_name
|
||||||
@@ -572,20 +804,33 @@ class SmallWebRTCTransport(BaseTransport):
|
|||||||
return self._output
|
return self._output
|
||||||
|
|
||||||
async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
|
async def send_image(self, frame: OutputImageRawFrame | SpriteFrame):
|
||||||
|
"""Send an image frame through the transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The image frame to send.
|
||||||
|
"""
|
||||||
if self._output:
|
if self._output:
|
||||||
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
||||||
|
|
||||||
async def send_audio(self, frame: OutputAudioRawFrame):
|
async def send_audio(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Send an audio frame through the transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to send.
|
||||||
|
"""
|
||||||
if self._output:
|
if self._output:
|
||||||
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
||||||
|
|
||||||
async def _on_app_message(self, message: Any):
|
async def _on_app_message(self, message: Any):
|
||||||
|
"""Handle incoming application messages."""
|
||||||
if self._input:
|
if self._input:
|
||||||
await self._input.push_app_message(message)
|
await self._input.push_app_message(message)
|
||||||
await self._call_event_handler("on_app_message", message)
|
await self._call_event_handler("on_app_message", message)
|
||||||
|
|
||||||
async def _on_client_connected(self, webrtc_connection):
|
async def _on_client_connected(self, webrtc_connection):
|
||||||
|
"""Handle client connection events."""
|
||||||
await self._call_event_handler("on_client_connected", webrtc_connection)
|
await self._call_event_handler("on_client_connected", webrtc_connection)
|
||||||
|
|
||||||
async def _on_client_disconnected(self, webrtc_connection):
|
async def _on_client_disconnected(self, webrtc_connection):
|
||||||
|
"""Handle client disconnection events."""
|
||||||
await self._call_event_handler("on_client_disconnected", webrtc_connection)
|
await self._call_event_handler("on_client_disconnected", webrtc_connection)
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Small WebRTC connection implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides a WebRTC connection implementation using aiortc,
|
||||||
|
with support for audio/video tracks, data channels, and signaling
|
||||||
|
for real-time communication applications.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import time
|
import time
|
||||||
@@ -35,36 +42,85 @@ VIDEO_TRANSCEIVER_INDEX = 1
|
|||||||
|
|
||||||
|
|
||||||
class TrackStatusMessage(BaseModel):
|
class TrackStatusMessage(BaseModel):
|
||||||
|
"""Message for updating track enabled/disabled status.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
type: Message type identifier.
|
||||||
|
receiver_index: Index of the track receiver to update.
|
||||||
|
enabled: Whether the track should be enabled or disabled.
|
||||||
|
"""
|
||||||
|
|
||||||
type: Literal["trackStatus"]
|
type: Literal["trackStatus"]
|
||||||
receiver_index: int
|
receiver_index: int
|
||||||
enabled: bool
|
enabled: bool
|
||||||
|
|
||||||
|
|
||||||
class RenegotiateMessage(BaseModel):
|
class RenegotiateMessage(BaseModel):
|
||||||
|
"""Message requesting WebRTC renegotiation.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
type: Message type identifier for renegotiation requests.
|
||||||
|
"""
|
||||||
|
|
||||||
type: Literal["renegotiate"] = "renegotiate"
|
type: Literal["renegotiate"] = "renegotiate"
|
||||||
|
|
||||||
|
|
||||||
class PeerLeftMessage(BaseModel):
|
class PeerLeftMessage(BaseModel):
|
||||||
|
"""Message indicating a peer has left the connection.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
type: Message type identifier for peer departure.
|
||||||
|
"""
|
||||||
|
|
||||||
type: Literal["peerLeft"] = "peerLeft"
|
type: Literal["peerLeft"] = "peerLeft"
|
||||||
|
|
||||||
|
|
||||||
class SignallingMessage:
|
class SignallingMessage:
|
||||||
|
"""Union types for signaling message handling.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
Inbound: Types of messages that can be received from peers.
|
||||||
|
outbound: Types of messages that can be sent to peers.
|
||||||
|
"""
|
||||||
|
|
||||||
Inbound = Union[TrackStatusMessage] # in case we need to add new messages in the future
|
Inbound = Union[TrackStatusMessage] # in case we need to add new messages in the future
|
||||||
outbound = Union[RenegotiateMessage]
|
outbound = Union[RenegotiateMessage]
|
||||||
|
|
||||||
|
|
||||||
class SmallWebRTCTrack:
|
class SmallWebRTCTrack:
|
||||||
|
"""Wrapper for WebRTC media tracks with enabled/disabled state management.
|
||||||
|
|
||||||
|
Provides additional functionality on top of aiortc MediaStreamTrack including
|
||||||
|
enable/disable control and frame discarding for audio and video streams.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, track: MediaStreamTrack):
|
def __init__(self, track: MediaStreamTrack):
|
||||||
|
"""Initialize the WebRTC track wrapper.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
track: The underlying MediaStreamTrack to wrap.
|
||||||
|
"""
|
||||||
self._track = track
|
self._track = track
|
||||||
self._enabled = True
|
self._enabled = True
|
||||||
|
|
||||||
def set_enabled(self, enabled: bool) -> None:
|
def set_enabled(self, enabled: bool) -> None:
|
||||||
|
"""Enable or disable the track.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
enabled: Whether the track should be enabled for receiving frames.
|
||||||
|
"""
|
||||||
self._enabled = enabled
|
self._enabled = enabled
|
||||||
|
|
||||||
def is_enabled(self) -> bool:
|
def is_enabled(self) -> bool:
|
||||||
|
"""Check if the track is currently enabled.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the track is enabled for receiving frames.
|
||||||
|
"""
|
||||||
return self._enabled
|
return self._enabled
|
||||||
|
|
||||||
async def discard_old_frames(self):
|
async def discard_old_frames(self):
|
||||||
|
"""Discard old frames from the track queue to reduce latency."""
|
||||||
remote_track = self._track
|
remote_track = self._track
|
||||||
if isinstance(remote_track, RemoteStreamTrack):
|
if isinstance(remote_track, RemoteStreamTrack):
|
||||||
if not hasattr(remote_track, "_queue") or not isinstance(
|
if not hasattr(remote_track, "_queue") or not isinstance(
|
||||||
@@ -78,11 +134,24 @@ class SmallWebRTCTrack:
|
|||||||
remote_track._queue.task_done()
|
remote_track._queue.task_done()
|
||||||
|
|
||||||
async def recv(self) -> Optional[Frame]:
|
async def recv(self) -> Optional[Frame]:
|
||||||
|
"""Receive the next frame from the track.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The next frame if the track is enabled, None otherwise.
|
||||||
|
"""
|
||||||
if not self._enabled:
|
if not self._enabled:
|
||||||
return None
|
return None
|
||||||
return await self._track.recv()
|
return await self._track.recv()
|
||||||
|
|
||||||
def __getattr__(self, name):
|
def __getattr__(self, name):
|
||||||
|
"""Forward attribute access to the underlying track.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
name: The attribute name to access.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The attribute value from the underlying track.
|
||||||
|
"""
|
||||||
# Forward other attribute/method calls to the underlying track
|
# Forward other attribute/method calls to the underlying track
|
||||||
return getattr(self._track, name)
|
return getattr(self._track, name)
|
||||||
|
|
||||||
@@ -92,7 +161,22 @@ IceServer = RTCIceServer
|
|||||||
|
|
||||||
|
|
||||||
class SmallWebRTCConnection(BaseObject):
|
class SmallWebRTCConnection(BaseObject):
|
||||||
|
"""WebRTC connection implementation using aiortc.
|
||||||
|
|
||||||
|
Provides WebRTC peer connection functionality including ICE server configuration,
|
||||||
|
track management, data channel communication, and connection state handling
|
||||||
|
for real-time audio/video communication.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, ice_servers: Optional[Union[List[str], List[IceServer]]] = None):
|
def __init__(self, ice_servers: Optional[Union[List[str], List[IceServer]]] = None):
|
||||||
|
"""Initialize the WebRTC connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
ice_servers: List of ICE servers as URLs or IceServer objects.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
TypeError: If ice_servers contains mixed types or unsupported types.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
if not ice_servers:
|
if not ice_servers:
|
||||||
self.ice_servers: List[IceServer] = []
|
self.ice_servers: List[IceServer] = []
|
||||||
@@ -126,13 +210,24 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def pc(self) -> RTCPeerConnection:
|
def pc(self) -> RTCPeerConnection:
|
||||||
|
"""Get the underlying RTCPeerConnection.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The aiortc RTCPeerConnection instance.
|
||||||
|
"""
|
||||||
return self._pc
|
return self._pc
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def pc_id(self) -> str:
|
def pc_id(self) -> str:
|
||||||
|
"""Get the peer connection identifier.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The unique identifier for this peer connection.
|
||||||
|
"""
|
||||||
return self._pc_id
|
return self._pc_id
|
||||||
|
|
||||||
def _initialize(self):
|
def _initialize(self):
|
||||||
|
"""Initialize the peer connection and associated components."""
|
||||||
logger.debug("Initializing new peer connection")
|
logger.debug("Initializing new peer connection")
|
||||||
rtc_config = RTCConfiguration(iceServers=self.ice_servers)
|
rtc_config = RTCConfiguration(iceServers=self.ice_servers)
|
||||||
|
|
||||||
@@ -147,6 +242,8 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
self._pending_app_messages = []
|
self._pending_app_messages = []
|
||||||
|
|
||||||
def _setup_listeners(self):
|
def _setup_listeners(self):
|
||||||
|
"""Set up event listeners for the peer connection."""
|
||||||
|
|
||||||
@self._pc.on("datachannel")
|
@self._pc.on("datachannel")
|
||||||
def on_datachannel(channel):
|
def on_datachannel(channel):
|
||||||
self._data_channel = channel
|
self._data_channel = channel
|
||||||
@@ -208,6 +305,7 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
await self._call_event_handler("track-ended", track)
|
await self._call_event_handler("track-ended", track)
|
||||||
|
|
||||||
async def _create_answer(self, sdp: str, type: str):
|
async def _create_answer(self, sdp: str, type: str):
|
||||||
|
"""Create an SDP answer for the given offer."""
|
||||||
offer = RTCSessionDescription(sdp=sdp, type=type)
|
offer = RTCSessionDescription(sdp=sdp, type=type)
|
||||||
await self._pc.setRemoteDescription(offer)
|
await self._pc.setRemoteDescription(offer)
|
||||||
|
|
||||||
@@ -223,9 +321,16 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
self._answer = self._pc.localDescription
|
self._answer = self._pc.localDescription
|
||||||
|
|
||||||
async def initialize(self, sdp: str, type: str):
|
async def initialize(self, sdp: str, type: str):
|
||||||
|
"""Initialize the connection with an SDP offer.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sdp: The SDP offer string.
|
||||||
|
type: The SDP type (usually "offer").
|
||||||
|
"""
|
||||||
await self._create_answer(sdp, type)
|
await self._create_answer(sdp, type)
|
||||||
|
|
||||||
async def connect(self):
|
async def connect(self):
|
||||||
|
"""Connect the WebRTC peer connection and handle initial setup."""
|
||||||
self._connect_invoked = True
|
self._connect_invoked = True
|
||||||
# If we already connected, trigger again the connected event
|
# If we already connected, trigger again the connected event
|
||||||
if self.is_connected():
|
if self.is_connected():
|
||||||
@@ -241,6 +346,13 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
self.ask_to_renegotiate()
|
self.ask_to_renegotiate()
|
||||||
|
|
||||||
async def renegotiate(self, sdp: str, type: str, restart_pc: bool = False):
|
async def renegotiate(self, sdp: str, type: str, restart_pc: bool = False):
|
||||||
|
"""Renegotiate the WebRTC connection with new parameters.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
sdp: The new SDP offer string.
|
||||||
|
type: The SDP type (usually "offer").
|
||||||
|
restart_pc: Whether to restart the peer connection entirely.
|
||||||
|
"""
|
||||||
logger.debug(f"Renegotiating {self._pc_id}")
|
logger.debug(f"Renegotiating {self._pc_id}")
|
||||||
|
|
||||||
if restart_pc:
|
if restart_pc:
|
||||||
@@ -264,6 +376,7 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
asyncio.create_task(delayed_task())
|
asyncio.create_task(delayed_task())
|
||||||
|
|
||||||
def force_transceivers_to_send_recv(self):
|
def force_transceivers_to_send_recv(self):
|
||||||
|
"""Force all transceivers to bidirectional send/receive mode."""
|
||||||
for transceiver in self._pc.getTransceivers():
|
for transceiver in self._pc.getTransceivers():
|
||||||
transceiver.direction = "sendrecv"
|
transceiver.direction = "sendrecv"
|
||||||
# logger.debug(
|
# logger.debug(
|
||||||
@@ -272,6 +385,11 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
# logger.debug(f"Sender track: {transceiver.sender.track}")
|
# logger.debug(f"Sender track: {transceiver.sender.track}")
|
||||||
|
|
||||||
def replace_audio_track(self, track):
|
def replace_audio_track(self, track):
|
||||||
|
"""Replace the audio track in the first transceiver.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
track: The new audio track to use for sending.
|
||||||
|
"""
|
||||||
logger.debug(f"Replacing audio track {track.kind}")
|
logger.debug(f"Replacing audio track {track.kind}")
|
||||||
# Transceivers always appear in creation-order for both peers
|
# Transceivers always appear in creation-order for both peers
|
||||||
# For now we are only considering that we are going to have 02 transceivers,
|
# For now we are only considering that we are going to have 02 transceivers,
|
||||||
@@ -283,6 +401,11 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
logger.warning("Audio transceiver not found. Cannot replace audio track.")
|
logger.warning("Audio transceiver not found. Cannot replace audio track.")
|
||||||
|
|
||||||
def replace_video_track(self, track):
|
def replace_video_track(self, track):
|
||||||
|
"""Replace the video track in the second transceiver.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
track: The new video track to use for sending.
|
||||||
|
"""
|
||||||
logger.debug(f"Replacing video track {track.kind}")
|
logger.debug(f"Replacing video track {track.kind}")
|
||||||
# Transceivers always appear in creation-order for both peers
|
# Transceivers always appear in creation-order for both peers
|
||||||
# For now we are only considering that we are going to have 02 transceivers,
|
# For now we are only considering that we are going to have 02 transceivers,
|
||||||
@@ -294,10 +417,12 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
logger.warning("Video transceiver not found. Cannot replace video track.")
|
logger.warning("Video transceiver not found. Cannot replace video track.")
|
||||||
|
|
||||||
async def disconnect(self):
|
async def disconnect(self):
|
||||||
|
"""Disconnect from the WebRTC peer connection."""
|
||||||
self.send_app_message({"type": SIGNALLING_TYPE, "message": PeerLeftMessage().model_dump()})
|
self.send_app_message({"type": SIGNALLING_TYPE, "message": PeerLeftMessage().model_dump()})
|
||||||
await self._close()
|
await self._close()
|
||||||
|
|
||||||
async def _close(self):
|
async def _close(self):
|
||||||
|
"""Close the peer connection and cleanup resources."""
|
||||||
if self._pc:
|
if self._pc:
|
||||||
await self._pc.close()
|
await self._pc.close()
|
||||||
self._message_queue.clear()
|
self._message_queue.clear()
|
||||||
@@ -305,6 +430,12 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
self._track_map = {}
|
self._track_map = {}
|
||||||
|
|
||||||
def get_answer(self):
|
def get_answer(self):
|
||||||
|
"""Get the SDP answer for the current connection.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary containing SDP answer, type, and peer connection ID,
|
||||||
|
or None if no answer is available.
|
||||||
|
"""
|
||||||
if not self._answer:
|
if not self._answer:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -315,6 +446,7 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
}
|
}
|
||||||
|
|
||||||
async def _handle_new_connection_state(self):
|
async def _handle_new_connection_state(self):
|
||||||
|
"""Handle changes in the peer connection state."""
|
||||||
state = self._pc.connectionState
|
state = self._pc.connectionState
|
||||||
if state == "connected" and not self._connect_invoked:
|
if state == "connected" and not self._connect_invoked:
|
||||||
# We are going to wait until the pipeline is ready before triggering the event
|
# We are going to wait until the pipeline is ready before triggering the event
|
||||||
@@ -328,7 +460,12 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
# Despite the fact that aiortc provides this listener, they don't have a status for "disconnected"
|
# Despite the fact that aiortc provides this listener, they don't have a status for "disconnected"
|
||||||
# So, there is no advantage in looking at self._pc.connectionState
|
# So, there is no advantage in looking at self._pc.connectionState
|
||||||
# That is why we are trying to keep our own state
|
# That is why we are trying to keep our own state
|
||||||
def is_connected(self):
|
def is_connected(self) -> bool:
|
||||||
|
"""Check if the WebRTC connection is currently active.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the connection is active and receiving data.
|
||||||
|
"""
|
||||||
# If the small webrtc transport has never invoked to connect
|
# If the small webrtc transport has never invoked to connect
|
||||||
# we are acting like if we are not connected
|
# we are acting like if we are not connected
|
||||||
if not self._connect_invoked:
|
if not self._connect_invoked:
|
||||||
@@ -342,6 +479,11 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
return (time.time() - self._last_received_time) < 3
|
return (time.time() - self._last_received_time) < 3
|
||||||
|
|
||||||
def audio_input_track(self):
|
def audio_input_track(self):
|
||||||
|
"""Get the audio input track wrapper.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
SmallWebRTCTrack wrapper for the audio track, or None if unavailable.
|
||||||
|
"""
|
||||||
if self._track_map.get(AUDIO_TRANSCEIVER_INDEX):
|
if self._track_map.get(AUDIO_TRANSCEIVER_INDEX):
|
||||||
return self._track_map[AUDIO_TRANSCEIVER_INDEX]
|
return self._track_map[AUDIO_TRANSCEIVER_INDEX]
|
||||||
|
|
||||||
@@ -359,6 +501,11 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
return audio_track
|
return audio_track
|
||||||
|
|
||||||
def video_input_track(self):
|
def video_input_track(self):
|
||||||
|
"""Get the video input track wrapper.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
SmallWebRTCTrack wrapper for the video track, or None if unavailable.
|
||||||
|
"""
|
||||||
if self._track_map.get(VIDEO_TRANSCEIVER_INDEX):
|
if self._track_map.get(VIDEO_TRANSCEIVER_INDEX):
|
||||||
return self._track_map[VIDEO_TRANSCEIVER_INDEX]
|
return self._track_map[VIDEO_TRANSCEIVER_INDEX]
|
||||||
|
|
||||||
@@ -376,6 +523,11 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
return video_track
|
return video_track
|
||||||
|
|
||||||
def send_app_message(self, message: Any):
|
def send_app_message(self, message: Any):
|
||||||
|
"""Send an application message through the data channel.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
message: The message to send (will be JSON serialized).
|
||||||
|
"""
|
||||||
json_message = json.dumps(message)
|
json_message = json.dumps(message)
|
||||||
if self._data_channel and self._data_channel.readyState == "open":
|
if self._data_channel and self._data_channel.readyState == "open":
|
||||||
self._data_channel.send(json_message)
|
self._data_channel.send(json_message)
|
||||||
@@ -384,6 +536,7 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
self._message_queue.append(json_message)
|
self._message_queue.append(json_message)
|
||||||
|
|
||||||
def ask_to_renegotiate(self):
|
def ask_to_renegotiate(self):
|
||||||
|
"""Request renegotiation of the WebRTC connection."""
|
||||||
if self._renegotiation_in_progress:
|
if self._renegotiation_in_progress:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -393,6 +546,7 @@ class SmallWebRTCConnection(BaseObject):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def _handle_signalling_message(self, message):
|
def _handle_signalling_message(self, message):
|
||||||
|
"""Handle incoming signaling messages."""
|
||||||
logger.debug(f"Signalling message received: {message}")
|
logger.debug(f"Signalling message received: {message}")
|
||||||
inbound_adapter = TypeAdapter(SignallingMessage.Inbound)
|
inbound_adapter = TypeAdapter(SignallingMessage.Inbound)
|
||||||
signalling_message = inbound_adapter.validate_python(message)
|
signalling_message = inbound_adapter.validate_python(message)
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""WebSocket client transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides a WebSocket client transport that enables bidirectional
|
||||||
|
communication over WebSocket connections, with support for audio streaming,
|
||||||
|
frame serialization, and connection management.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import io
|
import io
|
||||||
import time
|
import time
|
||||||
@@ -34,17 +41,38 @@ from pipecat.utils.asyncio.task_manager import BaseTaskManager
|
|||||||
|
|
||||||
|
|
||||||
class WebsocketClientParams(TransportParams):
|
class WebsocketClientParams(TransportParams):
|
||||||
|
"""Configuration parameters for WebSocket client transport.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
add_wav_header: Whether to add WAV headers to audio frames.
|
||||||
|
serializer: Frame serializer for encoding/decoding messages.
|
||||||
|
"""
|
||||||
|
|
||||||
add_wav_header: bool = True
|
add_wav_header: bool = True
|
||||||
serializer: Optional[FrameSerializer] = None
|
serializer: Optional[FrameSerializer] = None
|
||||||
|
|
||||||
|
|
||||||
class WebsocketClientCallbacks(BaseModel):
|
class WebsocketClientCallbacks(BaseModel):
|
||||||
|
"""Callback functions for WebSocket client events.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
on_connected: Called when WebSocket connection is established.
|
||||||
|
on_disconnected: Called when WebSocket connection is closed.
|
||||||
|
on_message: Called when a message is received from the WebSocket.
|
||||||
|
"""
|
||||||
|
|
||||||
on_connected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
|
on_connected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
|
||||||
on_disconnected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
|
on_disconnected: Callable[[websockets.WebSocketClientProtocol], Awaitable[None]]
|
||||||
on_message: Callable[[websockets.WebSocketClientProtocol, websockets.Data], Awaitable[None]]
|
on_message: Callable[[websockets.WebSocketClientProtocol, websockets.Data], Awaitable[None]]
|
||||||
|
|
||||||
|
|
||||||
class WebsocketClientSession:
|
class WebsocketClientSession:
|
||||||
|
"""Manages a WebSocket client connection session.
|
||||||
|
|
||||||
|
Handles connection lifecycle, message sending/receiving, and provides
|
||||||
|
callback mechanisms for connection events.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
uri: str,
|
uri: str,
|
||||||
@@ -52,6 +80,14 @@ class WebsocketClientSession:
|
|||||||
callbacks: WebsocketClientCallbacks,
|
callbacks: WebsocketClientCallbacks,
|
||||||
transport_name: str,
|
transport_name: str,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket client session.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
uri: The WebSocket URI to connect to.
|
||||||
|
params: Configuration parameters for the session.
|
||||||
|
callbacks: Callback functions for session events.
|
||||||
|
transport_name: Name of the parent transport for logging.
|
||||||
|
"""
|
||||||
self._uri = uri
|
self._uri = uri
|
||||||
self._params = params
|
self._params = params
|
||||||
self._callbacks = callbacks
|
self._callbacks = callbacks
|
||||||
@@ -63,6 +99,14 @@ class WebsocketClientSession:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def task_manager(self) -> BaseTaskManager:
|
def task_manager(self) -> BaseTaskManager:
|
||||||
|
"""Get the task manager for this session.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The task manager instance.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If task manager is not initialized.
|
||||||
|
"""
|
||||||
if not self._task_manager:
|
if not self._task_manager:
|
||||||
raise Exception(
|
raise Exception(
|
||||||
f"{self._transport_name}::WebsocketClientSession: TaskManager not initialized (pipeline not started?)"
|
f"{self._transport_name}::WebsocketClientSession: TaskManager not initialized (pipeline not started?)"
|
||||||
@@ -70,11 +114,17 @@ class WebsocketClientSession:
|
|||||||
return self._task_manager
|
return self._task_manager
|
||||||
|
|
||||||
async def setup(self, task_manager: BaseTaskManager):
|
async def setup(self, task_manager: BaseTaskManager):
|
||||||
|
"""Set up the session with a task manager.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task_manager: The task manager to use for session tasks.
|
||||||
|
"""
|
||||||
self._leave_counter += 1
|
self._leave_counter += 1
|
||||||
if not self._task_manager:
|
if not self._task_manager:
|
||||||
self._task_manager = task_manager
|
self._task_manager = task_manager
|
||||||
|
|
||||||
async def connect(self):
|
async def connect(self):
|
||||||
|
"""Connect to the WebSocket server."""
|
||||||
if self._websocket:
|
if self._websocket:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -89,6 +139,7 @@ class WebsocketClientSession:
|
|||||||
logger.error(f"Timeout connecting to {self._uri}")
|
logger.error(f"Timeout connecting to {self._uri}")
|
||||||
|
|
||||||
async def disconnect(self):
|
async def disconnect(self):
|
||||||
|
"""Disconnect from the WebSocket server."""
|
||||||
self._leave_counter -= 1
|
self._leave_counter -= 1
|
||||||
if not self._websocket or self._leave_counter > 0:
|
if not self._websocket or self._leave_counter > 0:
|
||||||
return
|
return
|
||||||
@@ -99,6 +150,11 @@ class WebsocketClientSession:
|
|||||||
self._websocket = None
|
self._websocket = None
|
||||||
|
|
||||||
async def send(self, message: websockets.Data):
|
async def send(self, message: websockets.Data):
|
||||||
|
"""Send a message through the WebSocket connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
message: The message data to send.
|
||||||
|
"""
|
||||||
try:
|
try:
|
||||||
if self._websocket:
|
if self._websocket:
|
||||||
await self._websocket.send(message)
|
await self._websocket.send(message)
|
||||||
@@ -106,6 +162,7 @@ class WebsocketClientSession:
|
|||||||
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
|
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
|
||||||
|
|
||||||
async def _client_task_handler(self):
|
async def _client_task_handler(self):
|
||||||
|
"""Handle incoming messages from the WebSocket connection."""
|
||||||
try:
|
try:
|
||||||
# Handle incoming messages
|
# Handle incoming messages
|
||||||
async for message in self._websocket:
|
async for message in self._websocket:
|
||||||
@@ -116,16 +173,30 @@ class WebsocketClientSession:
|
|||||||
await self._callbacks.on_disconnected(self._websocket)
|
await self._callbacks.on_disconnected(self._websocket)
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
|
"""String representation of the WebSocket client session."""
|
||||||
return f"{self._transport_name}::WebsocketClientSession"
|
return f"{self._transport_name}::WebsocketClientSession"
|
||||||
|
|
||||||
|
|
||||||
class WebsocketClientInputTransport(BaseInputTransport):
|
class WebsocketClientInputTransport(BaseInputTransport):
|
||||||
|
"""WebSocket client input transport for receiving frames.
|
||||||
|
|
||||||
|
Handles incoming WebSocket messages, deserializes them to frames,
|
||||||
|
and pushes them downstream in the processing pipeline.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: BaseTransport,
|
transport: BaseTransport,
|
||||||
session: WebsocketClientSession,
|
session: WebsocketClientSession,
|
||||||
params: WebsocketClientParams,
|
params: WebsocketClientParams,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket client input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
session: The WebSocket session to use for communication.
|
||||||
|
params: Configuration parameters for the transport.
|
||||||
|
"""
|
||||||
super().__init__(params)
|
super().__init__(params)
|
||||||
|
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
@@ -136,10 +207,20 @@ class WebsocketClientInputTransport(BaseInputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Set up the input transport with the frame processor setup.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
await self._session.setup(setup.task_manager)
|
await self._session.setup(setup.task_manager)
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the input transport and initialize the WebSocket connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -153,18 +234,35 @@ class WebsocketClientInputTransport(BaseInputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the input transport and disconnect from WebSocket.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._session.disconnect()
|
await self._session.disconnect()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the input transport and disconnect from WebSocket.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._session.disconnect()
|
await self._session.disconnect()
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Clean up the input transport resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def on_message(self, websocket, message):
|
async def on_message(self, websocket, message):
|
||||||
|
"""Handle incoming WebSocket messages.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
websocket: The WebSocket connection that received the message.
|
||||||
|
message: The received message data.
|
||||||
|
"""
|
||||||
if not self._params.serializer:
|
if not self._params.serializer:
|
||||||
return
|
return
|
||||||
frame = await self._params.serializer.deserialize(message)
|
frame = await self._params.serializer.deserialize(message)
|
||||||
@@ -177,12 +275,25 @@ class WebsocketClientInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class WebsocketClientOutputTransport(BaseOutputTransport):
|
class WebsocketClientOutputTransport(BaseOutputTransport):
|
||||||
|
"""WebSocket client output transport for sending frames.
|
||||||
|
|
||||||
|
Handles outgoing frames, serializes them for WebSocket transmission,
|
||||||
|
and manages audio streaming with proper timing simulation.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: BaseTransport,
|
transport: BaseTransport,
|
||||||
session: WebsocketClientSession,
|
session: WebsocketClientSession,
|
||||||
params: WebsocketClientParams,
|
params: WebsocketClientParams,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket client output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
session: The WebSocket session to use for communication.
|
||||||
|
params: Configuration parameters for the transport.
|
||||||
|
"""
|
||||||
super().__init__(params)
|
super().__init__(params)
|
||||||
|
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
@@ -201,10 +312,20 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Set up the output transport with the frame processor setup.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
await self._session.setup(setup.task_manager)
|
await self._session.setup(setup.task_manager)
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the output transport and initialize the WebSocket connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -219,21 +340,42 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the output transport and disconnect from WebSocket.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._session.disconnect()
|
await self._session.disconnect()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the output transport and disconnect from WebSocket.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._session.disconnect()
|
await self._session.disconnect()
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Clean up the output transport resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a transport message through the WebSocket.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The transport message frame to send.
|
||||||
|
"""
|
||||||
await self._write_frame(frame)
|
await self._write_frame(frame)
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the WebSocket with optional WAV header.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output audio frame to write.
|
||||||
|
"""
|
||||||
frame = OutputAudioRawFrame(
|
frame = OutputAudioRawFrame(
|
||||||
audio=frame.audio,
|
audio=frame.audio,
|
||||||
sample_rate=self.sample_rate,
|
sample_rate=self.sample_rate,
|
||||||
@@ -260,6 +402,7 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
|
|||||||
await self._write_audio_sleep()
|
await self._write_audio_sleep()
|
||||||
|
|
||||||
async def _write_frame(self, frame: Frame):
|
async def _write_frame(self, frame: Frame):
|
||||||
|
"""Write a frame to the WebSocket after serialization."""
|
||||||
if not self._params.serializer:
|
if not self._params.serializer:
|
||||||
return
|
return
|
||||||
payload = await self._params.serializer.serialize(frame)
|
payload = await self._params.serializer.serialize(frame)
|
||||||
@@ -267,6 +410,7 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
|
|||||||
await self._session.send(payload)
|
await self._session.send(payload)
|
||||||
|
|
||||||
async def _write_audio_sleep(self):
|
async def _write_audio_sleep(self):
|
||||||
|
"""Simulate audio playback timing with sleep delays."""
|
||||||
# Simulate a clock.
|
# Simulate a clock.
|
||||||
current_time = time.monotonic()
|
current_time = time.monotonic()
|
||||||
sleep_duration = max(0, self._next_send_time - current_time)
|
sleep_duration = max(0, self._next_send_time - current_time)
|
||||||
@@ -278,11 +422,23 @@ class WebsocketClientOutputTransport(BaseOutputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class WebsocketClientTransport(BaseTransport):
|
class WebsocketClientTransport(BaseTransport):
|
||||||
|
"""WebSocket client transport for bidirectional communication.
|
||||||
|
|
||||||
|
Provides a complete WebSocket client transport implementation with
|
||||||
|
input and output capabilities, connection management, and event handling.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
uri: str,
|
uri: str,
|
||||||
params: Optional[WebsocketClientParams] = None,
|
params: Optional[WebsocketClientParams] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket client transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
uri: The WebSocket URI to connect to.
|
||||||
|
params: Optional configuration parameters for the transport.
|
||||||
|
"""
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
self._params = params or WebsocketClientParams()
|
self._params = params or WebsocketClientParams()
|
||||||
@@ -304,21 +460,34 @@ class WebsocketClientTransport(BaseTransport):
|
|||||||
self._register_event_handler("on_disconnected")
|
self._register_event_handler("on_disconnected")
|
||||||
|
|
||||||
def input(self) -> WebsocketClientInputTransport:
|
def input(self) -> WebsocketClientInputTransport:
|
||||||
|
"""Get the input transport for receiving frames.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The WebSocket client input transport instance.
|
||||||
|
"""
|
||||||
if not self._input:
|
if not self._input:
|
||||||
self._input = WebsocketClientInputTransport(self, self._session, self._params)
|
self._input = WebsocketClientInputTransport(self, self._session, self._params)
|
||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> WebsocketClientOutputTransport:
|
def output(self) -> WebsocketClientOutputTransport:
|
||||||
|
"""Get the output transport for sending frames.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The WebSocket client output transport instance.
|
||||||
|
"""
|
||||||
if not self._output:
|
if not self._output:
|
||||||
self._output = WebsocketClientOutputTransport(self, self._session, self._params)
|
self._output = WebsocketClientOutputTransport(self, self._session, self._params)
|
||||||
return self._output
|
return self._output
|
||||||
|
|
||||||
async def _on_connected(self, websocket):
|
async def _on_connected(self, websocket):
|
||||||
|
"""Handle WebSocket connection established event."""
|
||||||
await self._call_event_handler("on_connected", websocket)
|
await self._call_event_handler("on_connected", websocket)
|
||||||
|
|
||||||
async def _on_disconnected(self, websocket):
|
async def _on_disconnected(self, websocket):
|
||||||
|
"""Handle WebSocket connection closed event."""
|
||||||
await self._call_event_handler("on_disconnected", websocket)
|
await self._call_event_handler("on_disconnected", websocket)
|
||||||
|
|
||||||
async def _on_message(self, websocket, message):
|
async def _on_message(self, websocket, message):
|
||||||
|
"""Handle incoming WebSocket message."""
|
||||||
if self._input:
|
if self._input:
|
||||||
await self._input.on_message(websocket, message)
|
await self._input.on_message(websocket, message)
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""WebSocket server transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides WebSocket server transport functionality for real-time
|
||||||
|
audio and data streaming, including client connection management, session
|
||||||
|
handling, and frame serialization.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import io
|
import io
|
||||||
import time
|
import time
|
||||||
@@ -39,12 +46,29 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
|
|
||||||
class WebsocketServerParams(TransportParams):
|
class WebsocketServerParams(TransportParams):
|
||||||
|
"""Configuration parameters for WebSocket server transport.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
add_wav_header: Whether to add WAV headers to audio frames.
|
||||||
|
serializer: Frame serializer for message encoding/decoding.
|
||||||
|
session_timeout: Timeout in seconds for client sessions.
|
||||||
|
"""
|
||||||
|
|
||||||
add_wav_header: bool = False
|
add_wav_header: bool = False
|
||||||
serializer: Optional[FrameSerializer] = None
|
serializer: Optional[FrameSerializer] = None
|
||||||
session_timeout: Optional[int] = None
|
session_timeout: Optional[int] = None
|
||||||
|
|
||||||
|
|
||||||
class WebsocketServerCallbacks(BaseModel):
|
class WebsocketServerCallbacks(BaseModel):
|
||||||
|
"""Callback functions for WebSocket server events.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
on_client_connected: Called when a client connects to the server.
|
||||||
|
on_client_disconnected: Called when a client disconnects from the server.
|
||||||
|
on_session_timeout: Called when a client session times out.
|
||||||
|
on_websocket_ready: Called when the WebSocket server is ready to accept connections.
|
||||||
|
"""
|
||||||
|
|
||||||
on_client_connected: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
|
on_client_connected: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
|
||||||
on_client_disconnected: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
|
on_client_disconnected: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
|
||||||
on_session_timeout: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
|
on_session_timeout: Callable[[websockets.WebSocketServerProtocol], Awaitable[None]]
|
||||||
@@ -52,6 +76,12 @@ class WebsocketServerCallbacks(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class WebsocketServerInputTransport(BaseInputTransport):
|
class WebsocketServerInputTransport(BaseInputTransport):
|
||||||
|
"""WebSocket server input transport for receiving client data.
|
||||||
|
|
||||||
|
Handles incoming WebSocket connections, message processing, and client
|
||||||
|
session management including timeout monitoring and connection lifecycle.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: BaseTransport,
|
transport: BaseTransport,
|
||||||
@@ -61,6 +91,16 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
callbacks: WebsocketServerCallbacks,
|
callbacks: WebsocketServerCallbacks,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket server input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
host: Host address to bind the WebSocket server to.
|
||||||
|
port: Port number to bind the WebSocket server to.
|
||||||
|
params: WebSocket server configuration parameters.
|
||||||
|
callbacks: Callback functions for WebSocket events.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
|
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
@@ -82,6 +122,11 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the WebSocket server and initialize components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -96,6 +141,11 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the WebSocket server and cleanup resources.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
self._stop_server_event.set()
|
self._stop_server_event.set()
|
||||||
if self._monitor_task:
|
if self._monitor_task:
|
||||||
@@ -106,6 +156,11 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
self._server_task = None
|
self._server_task = None
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the WebSocket server and stop all processing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
if self._monitor_task:
|
if self._monitor_task:
|
||||||
await self.cancel_task(self._monitor_task)
|
await self.cancel_task(self._monitor_task)
|
||||||
@@ -115,16 +170,19 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
self._server_task = None
|
self._server_task = None
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup resources and parent transport."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def _server_task_handler(self):
|
async def _server_task_handler(self):
|
||||||
|
"""Handle WebSocket server startup and client connections."""
|
||||||
logger.info(f"Starting websocket server on {self._host}:{self._port}")
|
logger.info(f"Starting websocket server on {self._host}:{self._port}")
|
||||||
async with websockets.serve(self._client_handler, self._host, self._port) as server:
|
async with websockets.serve(self._client_handler, self._host, self._port) as server:
|
||||||
await self._callbacks.on_websocket_ready()
|
await self._callbacks.on_websocket_ready()
|
||||||
await self._stop_server_event.wait()
|
await self._stop_server_event.wait()
|
||||||
|
|
||||||
async def _client_handler(self, websocket: websockets.WebSocketServerProtocol, path):
|
async def _client_handler(self, websocket: websockets.WebSocketServerProtocol, path):
|
||||||
|
"""Handle individual client connections and message processing."""
|
||||||
logger.info(f"New client connection from {websocket.remote_address}")
|
logger.info(f"New client connection from {websocket.remote_address}")
|
||||||
if self._websocket:
|
if self._websocket:
|
||||||
await self._websocket.close()
|
await self._websocket.close()
|
||||||
@@ -170,9 +228,7 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
async def _monitor_websocket(
|
async def _monitor_websocket(
|
||||||
self, websocket: websockets.WebSocketServerProtocol, session_timeout: int
|
self, websocket: websockets.WebSocketServerProtocol, session_timeout: int
|
||||||
):
|
):
|
||||||
"""Wait for session_timeout seconds, if the websocket is still open,
|
"""Monitor WebSocket connection for session timeout."""
|
||||||
trigger timeout event.
|
|
||||||
"""
|
|
||||||
try:
|
try:
|
||||||
await asyncio.sleep(session_timeout)
|
await asyncio.sleep(session_timeout)
|
||||||
if not websocket.closed:
|
if not websocket.closed:
|
||||||
@@ -183,7 +239,20 @@ class WebsocketServerInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class WebsocketServerOutputTransport(BaseOutputTransport):
|
class WebsocketServerOutputTransport(BaseOutputTransport):
|
||||||
|
"""WebSocket server output transport for sending data to clients.
|
||||||
|
|
||||||
|
Handles outgoing frame serialization, audio streaming with timing control,
|
||||||
|
and client connection management for WebSocket communication.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, transport: BaseTransport, params: WebsocketServerParams, **kwargs):
|
def __init__(self, transport: BaseTransport, params: WebsocketServerParams, **kwargs):
|
||||||
|
"""Initialize the WebSocket server output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
params: WebSocket server configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
|
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
@@ -203,12 +272,22 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def set_client_connection(self, websocket: Optional[websockets.WebSocketServerProtocol]):
|
async def set_client_connection(self, websocket: Optional[websockets.WebSocketServerProtocol]):
|
||||||
|
"""Set the active client WebSocket connection.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
websocket: The WebSocket connection to set as active, or None to clear.
|
||||||
|
"""
|
||||||
if self._websocket:
|
if self._websocket:
|
||||||
await self._websocket.close()
|
await self._websocket.close()
|
||||||
logger.warning("Only one client allowed, using new connection")
|
logger.warning("Only one client allowed, using new connection")
|
||||||
self._websocket = websocket
|
self._websocket = websocket
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the output transport and initialize components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -222,18 +301,35 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the output transport and send final frame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._write_frame(frame)
|
await self._write_frame(frame)
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the output transport and send cancellation frame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._write_frame(frame)
|
await self._write_frame(frame)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup resources and parent transport."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and handle interruption timing.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow in the pipeline.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
|
|
||||||
if isinstance(frame, StartInterruptionFrame):
|
if isinstance(frame, StartInterruptionFrame):
|
||||||
@@ -241,9 +337,19 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
|
|||||||
self._next_send_time = 0
|
self._next_send_time = 0
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a transport message frame to the client.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The transport message frame to send.
|
||||||
|
"""
|
||||||
await self._write_frame(frame)
|
await self._write_frame(frame)
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the WebSocket client with timing control.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The output audio frame to write.
|
||||||
|
"""
|
||||||
if not self._websocket:
|
if not self._websocket:
|
||||||
# Simulate audio playback with a sleep.
|
# Simulate audio playback with a sleep.
|
||||||
await self._write_audio_sleep()
|
await self._write_audio_sleep()
|
||||||
@@ -275,6 +381,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
|
|||||||
await self._write_audio_sleep()
|
await self._write_audio_sleep()
|
||||||
|
|
||||||
async def _write_frame(self, frame: Frame):
|
async def _write_frame(self, frame: Frame):
|
||||||
|
"""Serialize and send a frame to the WebSocket client."""
|
||||||
if not self._params.serializer:
|
if not self._params.serializer:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -286,6 +393,7 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
|
|||||||
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
|
logger.error(f"{self} exception sending data: {e.__class__.__name__} ({e})")
|
||||||
|
|
||||||
async def _write_audio_sleep(self):
|
async def _write_audio_sleep(self):
|
||||||
|
"""Simulate audio device timing by sleeping between audio chunks."""
|
||||||
# Simulate a clock.
|
# Simulate a clock.
|
||||||
current_time = time.monotonic()
|
current_time = time.monotonic()
|
||||||
sleep_duration = max(0, self._next_send_time - current_time)
|
sleep_duration = max(0, self._next_send_time - current_time)
|
||||||
@@ -297,6 +405,13 @@ class WebsocketServerOutputTransport(BaseOutputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class WebsocketServerTransport(BaseTransport):
|
class WebsocketServerTransport(BaseTransport):
|
||||||
|
"""WebSocket server transport for bidirectional real-time communication.
|
||||||
|
|
||||||
|
Provides a complete WebSocket server implementation with separate input and
|
||||||
|
output transports, client connection management, and event handling for
|
||||||
|
real-time audio and data streaming applications.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
params: WebsocketServerParams,
|
params: WebsocketServerParams,
|
||||||
@@ -305,6 +420,15 @@ class WebsocketServerTransport(BaseTransport):
|
|||||||
input_name: Optional[str] = None,
|
input_name: Optional[str] = None,
|
||||||
output_name: Optional[str] = None,
|
output_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the WebSocket server transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: WebSocket server configuration parameters.
|
||||||
|
host: Host address to bind the server to. Defaults to "localhost".
|
||||||
|
port: Port number to bind the server to. Defaults to 8765.
|
||||||
|
input_name: Optional name for the input processor.
|
||||||
|
output_name: Optional name for the output processor.
|
||||||
|
"""
|
||||||
super().__init__(input_name=input_name, output_name=output_name)
|
super().__init__(input_name=input_name, output_name=output_name)
|
||||||
self._host = host
|
self._host = host
|
||||||
self._port = port
|
self._port = port
|
||||||
@@ -328,6 +452,11 @@ class WebsocketServerTransport(BaseTransport):
|
|||||||
self._register_event_handler("on_websocket_ready")
|
self._register_event_handler("on_websocket_ready")
|
||||||
|
|
||||||
def input(self) -> WebsocketServerInputTransport:
|
def input(self) -> WebsocketServerInputTransport:
|
||||||
|
"""Get the input transport for receiving client data.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The WebSocket server input transport instance.
|
||||||
|
"""
|
||||||
if not self._input:
|
if not self._input:
|
||||||
self._input = WebsocketServerInputTransport(
|
self._input = WebsocketServerInputTransport(
|
||||||
self, self._host, self._port, self._params, self._callbacks, name=self._input_name
|
self, self._host, self._port, self._params, self._callbacks, name=self._input_name
|
||||||
@@ -335,6 +464,11 @@ class WebsocketServerTransport(BaseTransport):
|
|||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> WebsocketServerOutputTransport:
|
def output(self) -> WebsocketServerOutputTransport:
|
||||||
|
"""Get the output transport for sending data to clients.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The WebSocket server output transport instance.
|
||||||
|
"""
|
||||||
if not self._output:
|
if not self._output:
|
||||||
self._output = WebsocketServerOutputTransport(
|
self._output = WebsocketServerOutputTransport(
|
||||||
self, self._params, name=self._output_name
|
self, self._params, name=self._output_name
|
||||||
@@ -342,6 +476,7 @@ class WebsocketServerTransport(BaseTransport):
|
|||||||
return self._output
|
return self._output
|
||||||
|
|
||||||
async def _on_client_connected(self, websocket):
|
async def _on_client_connected(self, websocket):
|
||||||
|
"""Handle client connection events."""
|
||||||
if self._output:
|
if self._output:
|
||||||
await self._output.set_client_connection(websocket)
|
await self._output.set_client_connection(websocket)
|
||||||
await self._call_event_handler("on_client_connected", websocket)
|
await self._call_event_handler("on_client_connected", websocket)
|
||||||
@@ -349,6 +484,7 @@ class WebsocketServerTransport(BaseTransport):
|
|||||||
logger.error("A WebsocketServerTransport output is missing in the pipeline")
|
logger.error("A WebsocketServerTransport output is missing in the pipeline")
|
||||||
|
|
||||||
async def _on_client_disconnected(self, websocket):
|
async def _on_client_disconnected(self, websocket):
|
||||||
|
"""Handle client disconnection events."""
|
||||||
if self._output:
|
if self._output:
|
||||||
await self._output.set_client_connection(None)
|
await self._output.set_client_connection(None)
|
||||||
await self._call_event_handler("on_client_disconnected", websocket)
|
await self._call_event_handler("on_client_disconnected", websocket)
|
||||||
@@ -356,7 +492,9 @@ class WebsocketServerTransport(BaseTransport):
|
|||||||
logger.error("A WebsocketServerTransport output is missing in the pipeline")
|
logger.error("A WebsocketServerTransport output is missing in the pipeline")
|
||||||
|
|
||||||
async def _on_session_timeout(self, websocket):
|
async def _on_session_timeout(self, websocket):
|
||||||
|
"""Handle client session timeout events."""
|
||||||
await self._call_event_handler("on_session_timeout", websocket)
|
await self._call_event_handler("on_session_timeout", websocket)
|
||||||
|
|
||||||
async def _on_websocket_ready(self):
|
async def _on_websocket_ready(self):
|
||||||
|
"""Handle WebSocket server ready events."""
|
||||||
await self._call_event_handler("on_websocket_ready")
|
await self._call_event_handler("on_websocket_ready")
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -20,11 +20,11 @@ from pydantic import BaseModel, Field, ValidationError
|
|||||||
class DailyRoomSipParams(BaseModel):
|
class DailyRoomSipParams(BaseModel):
|
||||||
"""SIP configuration parameters for Daily rooms.
|
"""SIP configuration parameters for Daily rooms.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
display_name: Name shown for the SIP endpoint
|
display_name: Name shown for the SIP endpoint.
|
||||||
video: Whether video is enabled for SIP
|
video: Whether video is enabled for SIP.
|
||||||
sip_mode: SIP connection mode, typically 'dial-in'
|
sip_mode: SIP connection mode, typically 'dial-in'.
|
||||||
num_endpoints: Number of allowed SIP endpoints
|
num_endpoints: Number of allowed SIP endpoints.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
display_name: str = "sw-sip-dialin"
|
display_name: str = "sw-sip-dialin"
|
||||||
@@ -38,6 +38,12 @@ class RecordingsBucketConfig(BaseModel):
|
|||||||
|
|
||||||
Refer to the Daily API documentation for more information:
|
Refer to the Daily API documentation for more information:
|
||||||
https://docs.daily.co/guides/products/live-streaming-recording/storing-recordings-in-a-custom-s3-bucket
|
https://docs.daily.co/guides/products/live-streaming-recording/storing-recordings-in-a-custom-s3-bucket
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
bucket_name: Name of the S3 bucket for storing recordings.
|
||||||
|
bucket_region: AWS region where the S3 bucket is located.
|
||||||
|
assume_role_arn: ARN of the IAM role to assume for S3 access.
|
||||||
|
allow_api_access: Whether to allow API access to the recordings.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
bucket_name: str
|
bucket_name: str
|
||||||
@@ -49,21 +55,22 @@ class RecordingsBucketConfig(BaseModel):
|
|||||||
class DailyRoomProperties(BaseModel, extra="allow"):
|
class DailyRoomProperties(BaseModel, extra="allow"):
|
||||||
"""Properties for configuring a Daily room.
|
"""Properties for configuring a Daily room.
|
||||||
|
|
||||||
Attributes:
|
|
||||||
exp: Optional Unix epoch timestamp for room expiration (e.g., time.time() + 300 for 5 minutes)
|
|
||||||
enable_chat: Whether chat is enabled in the room
|
|
||||||
enable_prejoin_ui: Whether the pre-join UI is enabled
|
|
||||||
enable_emoji_reactions: Whether emoji reactions are enabled
|
|
||||||
eject_at_room_exp: Whether to remove participants when room expires
|
|
||||||
enable_dialout: Whether SIP dial-out is enabled
|
|
||||||
enable_recording: Recording settings ('cloud', 'local', 'raw-tracks')
|
|
||||||
geo: Geographic region for room
|
|
||||||
max_participants: Maximum number of participants allowed in the room
|
|
||||||
sip: SIP configuration parameters
|
|
||||||
sip_uri: SIP URI information returned by Daily
|
|
||||||
start_video_off: Whether video is off by default
|
|
||||||
|
|
||||||
Reference: https://docs.daily.co/reference/rest-api/rooms/create-room#properties
|
Reference: https://docs.daily.co/reference/rest-api/rooms/create-room#properties
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
exp: Optional Unix epoch timestamp for room expiration (e.g., time.time() + 300 for 5 minutes).
|
||||||
|
enable_chat: Whether chat is enabled in the room.
|
||||||
|
enable_prejoin_ui: Whether the pre-join UI is enabled.
|
||||||
|
enable_emoji_reactions: Whether emoji reactions are enabled.
|
||||||
|
eject_at_room_exp: Whether to remove participants when room expires.
|
||||||
|
enable_dialout: Whether SIP dial-out is enabled.
|
||||||
|
enable_recording: Recording settings ('cloud', 'local', 'raw-tracks').
|
||||||
|
geo: Geographic region for room.
|
||||||
|
max_participants: Maximum number of participants allowed in the room.
|
||||||
|
recordings_bucket: Configuration for custom S3 bucket recordings.
|
||||||
|
sip: SIP configuration parameters.
|
||||||
|
sip_uri: SIP URI information returned by Daily.
|
||||||
|
start_video_off: Whether video is off by default.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
exp: Optional[float] = None
|
exp: Optional[float] = None
|
||||||
@@ -85,7 +92,7 @@ class DailyRoomProperties(BaseModel, extra="allow"):
|
|||||||
"""Get the SIP endpoint URI if available.
|
"""Get the SIP endpoint URI if available.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str: SIP endpoint URI or empty string if not available
|
SIP endpoint URI or empty string if not available.
|
||||||
"""
|
"""
|
||||||
if not self.sip_uri:
|
if not self.sip_uri:
|
||||||
return ""
|
return ""
|
||||||
@@ -96,10 +103,10 @@ class DailyRoomProperties(BaseModel, extra="allow"):
|
|||||||
class DailyRoomParams(BaseModel):
|
class DailyRoomParams(BaseModel):
|
||||||
"""Parameters for creating a Daily room.
|
"""Parameters for creating a Daily room.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
name: Optional custom name for the room
|
name: Optional custom name for the room.
|
||||||
privacy: Room privacy setting ('private' or 'public')
|
privacy: Room privacy setting ('private' or 'public').
|
||||||
properties: Room configuration properties
|
properties: Room configuration properties.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
name: Optional[str] = None
|
name: Optional[str] = None
|
||||||
@@ -110,14 +117,14 @@ class DailyRoomParams(BaseModel):
|
|||||||
class DailyRoomObject(BaseModel):
|
class DailyRoomObject(BaseModel):
|
||||||
"""Represents a Daily room returned by the API.
|
"""Represents a Daily room returned by the API.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
id: Unique room identifier
|
id: Unique room identifier.
|
||||||
name: Room name
|
name: Room name.
|
||||||
api_created: Whether room was created via API
|
api_created: Whether room was created via API.
|
||||||
privacy: Room privacy setting ('private' or 'public')
|
privacy: Room privacy setting ('private' or 'public').
|
||||||
url: Full URL for joining the room
|
url: Full URL for joining the room.
|
||||||
created_at: Timestamp of room creation in ISO 8601 format (e.g., "2019-01-26T09:01:22.000Z").
|
created_at: Timestamp of room creation in ISO 8601 format (e.g., "2019-01-26T09:01:22.000Z").
|
||||||
config: Room configuration properties
|
config: Room configuration properties.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
id: str
|
id: str
|
||||||
@@ -134,71 +141,40 @@ class DailyMeetingTokenProperties(BaseModel):
|
|||||||
|
|
||||||
Refer to the Daily API documentation for more information:
|
Refer to the Daily API documentation for more information:
|
||||||
https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#properties
|
https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#properties
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
room_name: The room for which this token is valid. If not set, the token is valid for all rooms in your domain.
|
||||||
|
eject_at_token_exp: If True, the user will be ejected from the room when the token expires.
|
||||||
|
eject_after_elapsed: The number of seconds after which the user will be ejected from the room.
|
||||||
|
nbf: Not before timestamp - users cannot join with this token before this time.
|
||||||
|
exp: Expiration time (unix timestamp in seconds). Strongly recommended for security.
|
||||||
|
is_owner: If True, the token will grant owner privileges in the room.
|
||||||
|
user_name: The name of the user. This will be added to the token payload.
|
||||||
|
user_id: A unique identifier for the user. This will be added to the token payload.
|
||||||
|
enable_screenshare: If True, the user will be able to share their screen.
|
||||||
|
start_video_off: If True, the user's video will be turned off when they join the room.
|
||||||
|
start_audio_off: If True, the user's audio will be turned off when they join the room.
|
||||||
|
enable_recording: Recording settings for the token. Must be one of 'cloud', 'local' or 'raw-tracks'.
|
||||||
|
enable_prejoin_ui: If True, the user will see the prejoin UI before joining the room.
|
||||||
|
start_cloud_recording: Start cloud recording when the user joins the room.
|
||||||
|
permissions: Specifies the initial default permissions for a non-meeting-owner participant.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
room_name: Optional[str] = Field(
|
room_name: Optional[str] = None
|
||||||
default=None,
|
eject_at_token_exp: Optional[bool] = None
|
||||||
description="The room for which this token is valid. If not set, the token is valid for all rooms in your domain. You should always set room_name if using this token to control meeting access.",
|
eject_after_elapsed: Optional[int] = None
|
||||||
)
|
nbf: Optional[int] = None
|
||||||
|
exp: Optional[int] = None
|
||||||
eject_at_token_exp: Optional[bool] = Field(
|
is_owner: Optional[bool] = None
|
||||||
default=None,
|
user_name: Optional[str] = None
|
||||||
description="If `true`, the user will be ejected from the room when the token expires. Defaults to `false`.",
|
user_id: Optional[str] = None
|
||||||
)
|
enable_screenshare: Optional[bool] = None
|
||||||
eject_after_elapsed: Optional[int] = Field(
|
start_video_off: Optional[bool] = None
|
||||||
default=None,
|
start_audio_off: Optional[bool] = None
|
||||||
description="The number of seconds after which the user will be ejected from the room. If not provided, the user will not be ejected based on elapsed time.",
|
enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = None
|
||||||
)
|
enable_prejoin_ui: Optional[bool] = None
|
||||||
|
start_cloud_recording: Optional[bool] = None
|
||||||
nbf: Optional[int] = Field(
|
permissions: Optional[dict] = None
|
||||||
default=None,
|
|
||||||
description="Not before. This is a unix timestamp (seconds since the epoch.) Users cannot join a meeting in with this token before this time.",
|
|
||||||
)
|
|
||||||
|
|
||||||
exp: Optional[int] = Field(
|
|
||||||
default=None,
|
|
||||||
description="Expiration time (unix timestamp in seconds). We strongly recommend setting this value for security. If not set, the token will not expire. Refer docs for more info.",
|
|
||||||
)
|
|
||||||
is_owner: Optional[bool] = Field(
|
|
||||||
default=None,
|
|
||||||
description="If `true`, the token will grant owner privileges in the room. Defaults to `false`.",
|
|
||||||
)
|
|
||||||
user_name: Optional[str] = Field(
|
|
||||||
default=None,
|
|
||||||
description="The name of the user. This will be added to the token payload.",
|
|
||||||
)
|
|
||||||
user_id: Optional[str] = Field(
|
|
||||||
default=None,
|
|
||||||
description="A unique identifier for the user. This will be added to the token payload.",
|
|
||||||
)
|
|
||||||
enable_screenshare: Optional[bool] = Field(
|
|
||||||
default=None,
|
|
||||||
description="If `true`, the user will be able to share their screen. Defaults to `true`.",
|
|
||||||
)
|
|
||||||
start_video_off: Optional[bool] = Field(
|
|
||||||
default=None,
|
|
||||||
description="If `true`, the user's video will be turned off when they join the room. Defaults to `false`.",
|
|
||||||
)
|
|
||||||
start_audio_off: Optional[bool] = Field(
|
|
||||||
default=None,
|
|
||||||
description="If `true`, the user's audio will be turned off when they join the room. Defaults to `false`.",
|
|
||||||
)
|
|
||||||
enable_recording: Optional[Literal["cloud", "local", "raw-tracks"]] = Field(
|
|
||||||
default=None,
|
|
||||||
description="Recording settings for the token. Must be one of `cloud`, `local` or `raw-tracks`.",
|
|
||||||
)
|
|
||||||
enable_prejoin_ui: Optional[bool] = Field(
|
|
||||||
default=None,
|
|
||||||
description="If `true`, the user will see the prejoin UI before joining the room.",
|
|
||||||
)
|
|
||||||
start_cloud_recording: Optional[bool] = Field(
|
|
||||||
default=None,
|
|
||||||
description="Start cloud recording when the user joins the room. This can be used to always record and archive meetings, for example in a customer support context.",
|
|
||||||
)
|
|
||||||
permissions: Optional[dict] = Field(
|
|
||||||
default=None,
|
|
||||||
description="Specifies the initial default permissions for a non-meeting-owner participant joining a call.",
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class DailyMeetingTokenParams(BaseModel):
|
class DailyMeetingTokenParams(BaseModel):
|
||||||
@@ -206,6 +182,9 @@ class DailyMeetingTokenParams(BaseModel):
|
|||||||
|
|
||||||
Refer to the Daily API documentation for more information:
|
Refer to the Daily API documentation for more information:
|
||||||
https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#body-params
|
https://docs.daily.co/reference/rest-api/meeting-tokens/create-meeting-token#body-params
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
properties: Meeting token configuration properties.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
properties: DailyMeetingTokenProperties = Field(default_factory=DailyMeetingTokenProperties)
|
properties: DailyMeetingTokenProperties = Field(default_factory=DailyMeetingTokenProperties)
|
||||||
@@ -215,11 +194,6 @@ class DailyRESTHelper:
|
|||||||
"""Helper class for interacting with Daily's REST API.
|
"""Helper class for interacting with Daily's REST API.
|
||||||
|
|
||||||
Provides methods for creating, managing, and accessing Daily rooms.
|
Provides methods for creating, managing, and accessing Daily rooms.
|
||||||
|
|
||||||
Args:
|
|
||||||
daily_api_key: Your Daily API key
|
|
||||||
daily_api_url: Daily API base URL (e.g. "https://api.daily.co/v1")
|
|
||||||
aiohttp_session: Async HTTP session for making requests
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -229,7 +203,13 @@ class DailyRESTHelper:
|
|||||||
daily_api_url: str = "https://api.daily.co/v1",
|
daily_api_url: str = "https://api.daily.co/v1",
|
||||||
aiohttp_session: aiohttp.ClientSession,
|
aiohttp_session: aiohttp.ClientSession,
|
||||||
):
|
):
|
||||||
"""Initialize the Daily REST helper."""
|
"""Initialize the Daily REST helper.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
daily_api_key: Your Daily API key.
|
||||||
|
daily_api_url: Daily API base URL (e.g. "https://api.daily.co/v1").
|
||||||
|
aiohttp_session: Async HTTP session for making requests.
|
||||||
|
"""
|
||||||
self.daily_api_key = daily_api_key
|
self.daily_api_key = daily_api_key
|
||||||
self.daily_api_url = daily_api_url
|
self.daily_api_url = daily_api_url
|
||||||
self.aiohttp_session = aiohttp_session
|
self.aiohttp_session = aiohttp_session
|
||||||
@@ -238,10 +218,10 @@ class DailyRESTHelper:
|
|||||||
"""Extract room name from a Daily room URL.
|
"""Extract room name from a Daily room URL.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
room_url: Full Daily room URL
|
room_url: Full Daily room URL.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str: Room name portion of the URL
|
Room name portion of the URL.
|
||||||
"""
|
"""
|
||||||
return urlparse(room_url).path[1:]
|
return urlparse(room_url).path[1:]
|
||||||
|
|
||||||
@@ -249,10 +229,10 @@ class DailyRESTHelper:
|
|||||||
"""Get room details from a Daily room URL.
|
"""Get room details from a Daily room URL.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
room_url: Full Daily room URL
|
room_url: Full Daily room URL.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
DailyRoomObject: DailyRoomObject instance for the room
|
DailyRoomObject instance for the room.
|
||||||
"""
|
"""
|
||||||
room_name = self.get_name_from_url(room_url)
|
room_name = self.get_name_from_url(room_url)
|
||||||
return await self._get_room_from_name(room_name)
|
return await self._get_room_from_name(room_name)
|
||||||
@@ -261,13 +241,13 @@ class DailyRESTHelper:
|
|||||||
"""Create a new Daily room.
|
"""Create a new Daily room.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
params: Room configuration parameters
|
params: Room configuration parameters.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
DailyRoomObject: DailyRoomObject instance for the created room
|
DailyRoomObject instance for the created room.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: If room creation fails or response is invalid
|
Exception: If room creation fails or response is invalid.
|
||||||
"""
|
"""
|
||||||
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
|
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
|
||||||
json = params.model_dump(exclude_none=True)
|
json = params.model_dump(exclude_none=True)
|
||||||
@@ -298,19 +278,19 @@ class DailyRESTHelper:
|
|||||||
"""Generate a meeting token for user to join a Daily room.
|
"""Generate a meeting token for user to join a Daily room.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
room_url: Daily room URL
|
room_url: Daily room URL.
|
||||||
expiry_time: Token validity duration in seconds (default: 1 hour)
|
expiry_time: Token validity duration in seconds (default: 1 hour).
|
||||||
eject_at_token_exp: Whether to eject user when token expires
|
eject_at_token_exp: Whether to eject user when token expires.
|
||||||
owner: Whether token has owner privileges
|
owner: Whether token has owner privileges.
|
||||||
params: Optional additional token properties. Note that room_name,
|
params: Optional additional token properties. Note that room_name,
|
||||||
exp, and is_owner will be set based on the other function
|
exp, and is_owner will be set based on the other function
|
||||||
parameters regardless of values in params.
|
parameters regardless of values in params.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
str: Meeting token
|
Meeting token.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: If token generation fails or room URL is missing
|
Exception: If token generation fails or room URL is missing.
|
||||||
"""
|
"""
|
||||||
if not room_url:
|
if not room_url:
|
||||||
raise Exception(
|
raise Exception(
|
||||||
@@ -355,10 +335,10 @@ class DailyRESTHelper:
|
|||||||
"""Delete a room using its URL.
|
"""Delete a room using its URL.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
room_url: Daily room URL
|
room_url: Daily room URL.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
bool: True if deletion was successful
|
True if deletion was successful.
|
||||||
"""
|
"""
|
||||||
room_name = self.get_name_from_url(room_url)
|
room_name = self.get_name_from_url(room_url)
|
||||||
return await self.delete_room_by_name(room_name)
|
return await self.delete_room_by_name(room_name)
|
||||||
@@ -367,13 +347,13 @@ class DailyRESTHelper:
|
|||||||
"""Delete a room using its name.
|
"""Delete a room using its name.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
room_name: Name of the room to delete
|
room_name: Name of the room to delete.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
bool: True if deletion was successful
|
True if deletion was successful.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
Exception: If deletion fails (excluding 404 Not Found)
|
Exception: If deletion fails (excluding 404 Not Found).
|
||||||
"""
|
"""
|
||||||
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
|
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
|
||||||
async with self.aiohttp_session.delete(
|
async with self.aiohttp_session.delete(
|
||||||
@@ -386,17 +366,7 @@ class DailyRESTHelper:
|
|||||||
return True
|
return True
|
||||||
|
|
||||||
async def _get_room_from_name(self, room_name: str) -> DailyRoomObject:
|
async def _get_room_from_name(self, room_name: str) -> DailyRoomObject:
|
||||||
"""Internal method to get room details by name.
|
"""Internal method to get room details by name."""
|
||||||
|
|
||||||
Args:
|
|
||||||
room_name: Name of the room
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
DailyRoomObject: DailyRoomObject instance for the room
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
Exception: If room is not found or response is invalid
|
|
||||||
"""
|
|
||||||
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
|
headers = {"Authorization": f"Bearer {self.daily_api_key}"}
|
||||||
async with self.aiohttp_session.get(
|
async with self.aiohttp_session.get(
|
||||||
f"{self.daily_api_url}/rooms/{room_name}", headers=headers
|
f"{self.daily_api_url}/rooms/{room_name}", headers=headers
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""LiveKit transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides comprehensive LiveKit real-time communication integration
|
||||||
|
including audio streaming, data messaging, participant management, and room
|
||||||
|
event handling for conversational AI applications.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any, Awaitable, Callable, List, Optional
|
from typing import Any, Awaitable, Callable, List, Optional
|
||||||
@@ -41,19 +48,49 @@ except ModuleNotFoundError as e:
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class LiveKitTransportMessageFrame(TransportMessageFrame):
|
class LiveKitTransportMessageFrame(TransportMessageFrame):
|
||||||
|
"""Frame for transport messages in LiveKit rooms.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
participant_id: Optional ID of the participant this message is for/from.
|
||||||
|
"""
|
||||||
|
|
||||||
participant_id: Optional[str] = None
|
participant_id: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class LiveKitTransportMessageUrgentFrame(TransportMessageUrgentFrame):
|
class LiveKitTransportMessageUrgentFrame(TransportMessageUrgentFrame):
|
||||||
|
"""Frame for urgent transport messages in LiveKit rooms.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
participant_id: Optional ID of the participant this message is for/from.
|
||||||
|
"""
|
||||||
|
|
||||||
participant_id: Optional[str] = None
|
participant_id: Optional[str] = None
|
||||||
|
|
||||||
|
|
||||||
class LiveKitParams(TransportParams):
|
class LiveKitParams(TransportParams):
|
||||||
|
"""Configuration parameters for LiveKit transport.
|
||||||
|
|
||||||
|
Inherits all parameters from TransportParams without additional configuration.
|
||||||
|
"""
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class LiveKitCallbacks(BaseModel):
|
class LiveKitCallbacks(BaseModel):
|
||||||
|
"""Callback handlers for LiveKit events.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
on_connected: Called when connected to the LiveKit room.
|
||||||
|
on_disconnected: Called when disconnected from the LiveKit room.
|
||||||
|
on_participant_connected: Called when a participant joins the room.
|
||||||
|
on_participant_disconnected: Called when a participant leaves the room.
|
||||||
|
on_audio_track_subscribed: Called when an audio track is subscribed.
|
||||||
|
on_audio_track_unsubscribed: Called when an audio track is unsubscribed.
|
||||||
|
on_data_received: Called when data is received from a participant.
|
||||||
|
on_first_participant_joined: Called when the first participant joins.
|
||||||
|
"""
|
||||||
|
|
||||||
on_connected: Callable[[], Awaitable[None]]
|
on_connected: Callable[[], Awaitable[None]]
|
||||||
on_disconnected: Callable[[], Awaitable[None]]
|
on_disconnected: Callable[[], Awaitable[None]]
|
||||||
on_participant_connected: Callable[[str], Awaitable[None]]
|
on_participant_connected: Callable[[str], Awaitable[None]]
|
||||||
@@ -65,6 +102,12 @@ class LiveKitCallbacks(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class LiveKitTransportClient:
|
class LiveKitTransportClient:
|
||||||
|
"""Core client for interacting with LiveKit rooms.
|
||||||
|
|
||||||
|
Manages the connection to LiveKit rooms and handles all low-level API interactions
|
||||||
|
including room management, audio streaming, data messaging, and event handling.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
url: str,
|
url: str,
|
||||||
@@ -74,6 +117,16 @@ class LiveKitTransportClient:
|
|||||||
callbacks: LiveKitCallbacks,
|
callbacks: LiveKitCallbacks,
|
||||||
transport_name: str,
|
transport_name: str,
|
||||||
):
|
):
|
||||||
|
"""Initialize the LiveKit transport client.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: LiveKit server URL to connect to.
|
||||||
|
token: Authentication token for the room.
|
||||||
|
room_name: Name of the LiveKit room to join.
|
||||||
|
params: Configuration parameters for the transport.
|
||||||
|
callbacks: Event callback handlers.
|
||||||
|
transport_name: Name identifier for the transport.
|
||||||
|
"""
|
||||||
self._url = url
|
self._url = url
|
||||||
self._token = token
|
self._token = token
|
||||||
self._room_name = room_name
|
self._room_name = room_name
|
||||||
@@ -93,15 +146,33 @@ class LiveKitTransportClient:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def participant_id(self) -> str:
|
def participant_id(self) -> str:
|
||||||
|
"""Get the participant ID for this client.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The participant ID assigned by LiveKit.
|
||||||
|
"""
|
||||||
return self._participant_id
|
return self._participant_id
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def room(self) -> rtc.Room:
|
def room(self) -> rtc.Room:
|
||||||
|
"""Get the LiveKit room instance.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The LiveKit room object.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If room object is not available.
|
||||||
|
"""
|
||||||
if not self._room:
|
if not self._room:
|
||||||
raise Exception(f"{self}: missing room object (pipeline not started?)")
|
raise Exception(f"{self}: missing room object (pipeline not started?)")
|
||||||
return self._room
|
return self._room
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Setup the client with task manager and room initialization.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
if self._task_manager:
|
if self._task_manager:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -118,13 +189,20 @@ class LiveKitTransportClient:
|
|||||||
self.room.on("disconnected")(self._on_disconnected_wrapper)
|
self.room.on("disconnected")(self._on_disconnected_wrapper)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup client resources."""
|
||||||
await self.disconnect()
|
await self.disconnect()
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the client and initialize audio components.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
|
self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate
|
||||||
|
|
||||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
|
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10))
|
||||||
async def connect(self):
|
async def connect(self):
|
||||||
|
"""Connect to the LiveKit room with retry logic."""
|
||||||
if self._connected:
|
if self._connected:
|
||||||
# Increment disconnect counter if already connected.
|
# Increment disconnect counter if already connected.
|
||||||
self._disconnect_counter += 1
|
self._disconnect_counter += 1
|
||||||
@@ -168,6 +246,7 @@ class LiveKitTransportClient:
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
async def disconnect(self):
|
async def disconnect(self):
|
||||||
|
"""Disconnect from the LiveKit room."""
|
||||||
# Decrement leave counter when leaving.
|
# Decrement leave counter when leaving.
|
||||||
self._disconnect_counter -= 1
|
self._disconnect_counter -= 1
|
||||||
|
|
||||||
@@ -181,6 +260,12 @@ class LiveKitTransportClient:
|
|||||||
await self._callbacks.on_disconnected()
|
await self._callbacks.on_disconnected()
|
||||||
|
|
||||||
async def send_data(self, data: bytes, participant_id: Optional[str] = None):
|
async def send_data(self, data: bytes, participant_id: Optional[str] = None):
|
||||||
|
"""Send data to participants in the room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data: The data bytes to send.
|
||||||
|
participant_id: Optional specific participant to send to.
|
||||||
|
"""
|
||||||
if not self._connected:
|
if not self._connected:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -195,6 +280,11 @@ class LiveKitTransportClient:
|
|||||||
logger.error(f"Error sending data: {e}")
|
logger.error(f"Error sending data: {e}")
|
||||||
|
|
||||||
async def publish_audio(self, audio_frame: rtc.AudioFrame):
|
async def publish_audio(self, audio_frame: rtc.AudioFrame):
|
||||||
|
"""Publish an audio frame to the room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
audio_frame: The LiveKit audio frame to publish.
|
||||||
|
"""
|
||||||
if not self._connected or not self._audio_source:
|
if not self._connected or not self._audio_source:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -204,9 +294,22 @@ class LiveKitTransportClient:
|
|||||||
logger.error(f"Error publishing audio: {e}")
|
logger.error(f"Error publishing audio: {e}")
|
||||||
|
|
||||||
def get_participants(self) -> List[str]:
|
def get_participants(self) -> List[str]:
|
||||||
|
"""Get list of participant IDs in the room.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of participant IDs.
|
||||||
|
"""
|
||||||
return [p.sid for p in self.room.remote_participants.values()]
|
return [p.sid for p in self.room.remote_participants.values()]
|
||||||
|
|
||||||
async def get_participant_metadata(self, participant_id: str) -> dict:
|
async def get_participant_metadata(self, participant_id: str) -> dict:
|
||||||
|
"""Get metadata for a specific participant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to get metadata for.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary containing participant metadata.
|
||||||
|
"""
|
||||||
participant = self.room.remote_participants.get(participant_id)
|
participant = self.room.remote_participants.get(participant_id)
|
||||||
if participant:
|
if participant:
|
||||||
return {
|
return {
|
||||||
@@ -218,9 +321,19 @@ class LiveKitTransportClient:
|
|||||||
return {}
|
return {}
|
||||||
|
|
||||||
async def set_participant_metadata(self, metadata: str):
|
async def set_participant_metadata(self, metadata: str):
|
||||||
|
"""Set metadata for the local participant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
metadata: Metadata string to set.
|
||||||
|
"""
|
||||||
await self.room.local_participant.set_metadata(metadata)
|
await self.room.local_participant.set_metadata(metadata)
|
||||||
|
|
||||||
async def mute_participant(self, participant_id: str):
|
async def mute_participant(self, participant_id: str):
|
||||||
|
"""Mute a specific participant's audio tracks.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to mute.
|
||||||
|
"""
|
||||||
participant = self.room.remote_participants.get(participant_id)
|
participant = self.room.remote_participants.get(participant_id)
|
||||||
if participant:
|
if participant:
|
||||||
for track in participant.tracks.values():
|
for track in participant.tracks.values():
|
||||||
@@ -228,6 +341,11 @@ class LiveKitTransportClient:
|
|||||||
await track.set_enabled(False)
|
await track.set_enabled(False)
|
||||||
|
|
||||||
async def unmute_participant(self, participant_id: str):
|
async def unmute_participant(self, participant_id: str):
|
||||||
|
"""Unmute a specific participant's audio tracks.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to unmute.
|
||||||
|
"""
|
||||||
participant = self.room.remote_participants.get(participant_id)
|
participant = self.room.remote_participants.get(participant_id)
|
||||||
if participant:
|
if participant:
|
||||||
for track in participant.tracks.values():
|
for track in participant.tracks.values():
|
||||||
@@ -236,12 +354,14 @@ class LiveKitTransportClient:
|
|||||||
|
|
||||||
# Wrapper methods for event handlers
|
# Wrapper methods for event handlers
|
||||||
def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant):
|
def _on_participant_connected_wrapper(self, participant: rtc.RemoteParticipant):
|
||||||
|
"""Wrapper for participant connected events."""
|
||||||
self._task_manager.create_task(
|
self._task_manager.create_task(
|
||||||
self._async_on_participant_connected(participant),
|
self._async_on_participant_connected(participant),
|
||||||
f"{self}::_async_on_participant_connected",
|
f"{self}::_async_on_participant_connected",
|
||||||
)
|
)
|
||||||
|
|
||||||
def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant):
|
def _on_participant_disconnected_wrapper(self, participant: rtc.RemoteParticipant):
|
||||||
|
"""Wrapper for participant disconnected events."""
|
||||||
self._task_manager.create_task(
|
self._task_manager.create_task(
|
||||||
self._async_on_participant_disconnected(participant),
|
self._async_on_participant_disconnected(participant),
|
||||||
f"{self}::_async_on_participant_disconnected",
|
f"{self}::_async_on_participant_disconnected",
|
||||||
@@ -253,6 +373,7 @@ class LiveKitTransportClient:
|
|||||||
publication: rtc.RemoteTrackPublication,
|
publication: rtc.RemoteTrackPublication,
|
||||||
participant: rtc.RemoteParticipant,
|
participant: rtc.RemoteParticipant,
|
||||||
):
|
):
|
||||||
|
"""Wrapper for track subscribed events."""
|
||||||
self._task_manager.create_task(
|
self._task_manager.create_task(
|
||||||
self._async_on_track_subscribed(track, publication, participant),
|
self._async_on_track_subscribed(track, publication, participant),
|
||||||
f"{self}::_async_on_track_subscribed",
|
f"{self}::_async_on_track_subscribed",
|
||||||
@@ -264,27 +385,32 @@ class LiveKitTransportClient:
|
|||||||
publication: rtc.RemoteTrackPublication,
|
publication: rtc.RemoteTrackPublication,
|
||||||
participant: rtc.RemoteParticipant,
|
participant: rtc.RemoteParticipant,
|
||||||
):
|
):
|
||||||
|
"""Wrapper for track unsubscribed events."""
|
||||||
self._task_manager.create_task(
|
self._task_manager.create_task(
|
||||||
self._async_on_track_unsubscribed(track, publication, participant),
|
self._async_on_track_unsubscribed(track, publication, participant),
|
||||||
f"{self}::_async_on_track_unsubscribed",
|
f"{self}::_async_on_track_unsubscribed",
|
||||||
)
|
)
|
||||||
|
|
||||||
def _on_data_received_wrapper(self, data: rtc.DataPacket):
|
def _on_data_received_wrapper(self, data: rtc.DataPacket):
|
||||||
|
"""Wrapper for data received events."""
|
||||||
self._task_manager.create_task(
|
self._task_manager.create_task(
|
||||||
self._async_on_data_received(data),
|
self._async_on_data_received(data),
|
||||||
f"{self}::_async_on_data_received",
|
f"{self}::_async_on_data_received",
|
||||||
)
|
)
|
||||||
|
|
||||||
def _on_connected_wrapper(self):
|
def _on_connected_wrapper(self):
|
||||||
|
"""Wrapper for connected events."""
|
||||||
self._task_manager.create_task(self._async_on_connected(), f"{self}::_async_on_connected")
|
self._task_manager.create_task(self._async_on_connected(), f"{self}::_async_on_connected")
|
||||||
|
|
||||||
def _on_disconnected_wrapper(self):
|
def _on_disconnected_wrapper(self):
|
||||||
|
"""Wrapper for disconnected events."""
|
||||||
self._task_manager.create_task(
|
self._task_manager.create_task(
|
||||||
self._async_on_disconnected(), f"{self}::_async_on_disconnected"
|
self._async_on_disconnected(), f"{self}::_async_on_disconnected"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Async methods for event handling
|
# Async methods for event handling
|
||||||
async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant):
|
async def _async_on_participant_connected(self, participant: rtc.RemoteParticipant):
|
||||||
|
"""Handle participant connected events."""
|
||||||
logger.info(f"Participant connected: {participant.identity}")
|
logger.info(f"Participant connected: {participant.identity}")
|
||||||
await self._callbacks.on_participant_connected(participant.sid)
|
await self._callbacks.on_participant_connected(participant.sid)
|
||||||
if not self._other_participant_has_joined:
|
if not self._other_participant_has_joined:
|
||||||
@@ -292,6 +418,7 @@ class LiveKitTransportClient:
|
|||||||
await self._callbacks.on_first_participant_joined(participant.sid)
|
await self._callbacks.on_first_participant_joined(participant.sid)
|
||||||
|
|
||||||
async def _async_on_participant_disconnected(self, participant: rtc.RemoteParticipant):
|
async def _async_on_participant_disconnected(self, participant: rtc.RemoteParticipant):
|
||||||
|
"""Handle participant disconnected events."""
|
||||||
logger.info(f"Participant disconnected: {participant.identity}")
|
logger.info(f"Participant disconnected: {participant.identity}")
|
||||||
await self._callbacks.on_participant_disconnected(participant.sid)
|
await self._callbacks.on_participant_disconnected(participant.sid)
|
||||||
if len(self.get_participants()) == 0:
|
if len(self.get_participants()) == 0:
|
||||||
@@ -303,6 +430,7 @@ class LiveKitTransportClient:
|
|||||||
publication: rtc.RemoteTrackPublication,
|
publication: rtc.RemoteTrackPublication,
|
||||||
participant: rtc.RemoteParticipant,
|
participant: rtc.RemoteParticipant,
|
||||||
):
|
):
|
||||||
|
"""Handle track subscribed events."""
|
||||||
if track.kind == rtc.TrackKind.KIND_AUDIO:
|
if track.kind == rtc.TrackKind.KIND_AUDIO:
|
||||||
logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}")
|
logger.info(f"Audio track subscribed: {track.sid} from participant {participant.sid}")
|
||||||
self._audio_tracks[participant.sid] = track
|
self._audio_tracks[participant.sid] = track
|
||||||
@@ -318,22 +446,27 @@ class LiveKitTransportClient:
|
|||||||
publication: rtc.RemoteTrackPublication,
|
publication: rtc.RemoteTrackPublication,
|
||||||
participant: rtc.RemoteParticipant,
|
participant: rtc.RemoteParticipant,
|
||||||
):
|
):
|
||||||
|
"""Handle track unsubscribed events."""
|
||||||
logger.info(f"Track unsubscribed: {publication.sid} from {participant.identity}")
|
logger.info(f"Track unsubscribed: {publication.sid} from {participant.identity}")
|
||||||
if track.kind == rtc.TrackKind.KIND_AUDIO:
|
if track.kind == rtc.TrackKind.KIND_AUDIO:
|
||||||
await self._callbacks.on_audio_track_unsubscribed(participant.sid)
|
await self._callbacks.on_audio_track_unsubscribed(participant.sid)
|
||||||
|
|
||||||
async def _async_on_data_received(self, data: rtc.DataPacket):
|
async def _async_on_data_received(self, data: rtc.DataPacket):
|
||||||
|
"""Handle data received events."""
|
||||||
await self._callbacks.on_data_received(data.data, data.participant.sid)
|
await self._callbacks.on_data_received(data.data, data.participant.sid)
|
||||||
|
|
||||||
async def _async_on_connected(self):
|
async def _async_on_connected(self):
|
||||||
|
"""Handle connected events."""
|
||||||
await self._callbacks.on_connected()
|
await self._callbacks.on_connected()
|
||||||
|
|
||||||
async def _async_on_disconnected(self, reason=None):
|
async def _async_on_disconnected(self, reason=None):
|
||||||
|
"""Handle disconnected events."""
|
||||||
self._connected = False
|
self._connected = False
|
||||||
logger.info(f"Disconnected from {self._room_name}. Reason: {reason}")
|
logger.info(f"Disconnected from {self._room_name}. Reason: {reason}")
|
||||||
await self._callbacks.on_disconnected()
|
await self._callbacks.on_disconnected()
|
||||||
|
|
||||||
async def _process_audio_stream(self, audio_stream: rtc.AudioStream, participant_id: str):
|
async def _process_audio_stream(self, audio_stream: rtc.AudioStream, participant_id: str):
|
||||||
|
"""Process incoming audio stream from a participant."""
|
||||||
logger.info(f"Started processing audio stream for participant {participant_id}")
|
logger.info(f"Started processing audio stream for participant {participant_id}")
|
||||||
async for event in audio_stream:
|
async for event in audio_stream:
|
||||||
if isinstance(event, rtc.AudioFrameEvent):
|
if isinstance(event, rtc.AudioFrameEvent):
|
||||||
@@ -342,15 +475,23 @@ class LiveKitTransportClient:
|
|||||||
logger.warning(f"Received unexpected event type: {type(event)}")
|
logger.warning(f"Received unexpected event type: {type(event)}")
|
||||||
|
|
||||||
async def get_next_audio_frame(self):
|
async def get_next_audio_frame(self):
|
||||||
|
"""Get the next audio frame from the queue."""
|
||||||
while True:
|
while True:
|
||||||
frame, participant_id = await self._audio_queue.get()
|
frame, participant_id = await self._audio_queue.get()
|
||||||
yield frame, participant_id
|
yield frame, participant_id
|
||||||
|
|
||||||
def __str__(self):
|
def __str__(self):
|
||||||
|
"""String representation of the LiveKit transport client."""
|
||||||
return f"{self._transport_name}::LiveKitTransportClient"
|
return f"{self._transport_name}::LiveKitTransportClient"
|
||||||
|
|
||||||
|
|
||||||
class LiveKitInputTransport(BaseInputTransport):
|
class LiveKitInputTransport(BaseInputTransport):
|
||||||
|
"""Handles incoming media streams and events from LiveKit rooms.
|
||||||
|
|
||||||
|
Processes incoming audio streams from room participants and forwards them
|
||||||
|
as Pipecat frames, including audio resampling and VAD integration.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: BaseTransport,
|
transport: BaseTransport,
|
||||||
@@ -358,6 +499,14 @@ class LiveKitInputTransport(BaseInputTransport):
|
|||||||
params: LiveKitParams,
|
params: LiveKitParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the LiveKit input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
client: LiveKitTransportClient instance.
|
||||||
|
params: Configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._client = client
|
self._client = client
|
||||||
@@ -371,9 +520,19 @@ class LiveKitInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def vad_analyzer(self) -> Optional[VADAnalyzer]:
|
def vad_analyzer(self) -> Optional[VADAnalyzer]:
|
||||||
|
"""Get the Voice Activity Detection analyzer.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The VAD analyzer instance if configured.
|
||||||
|
"""
|
||||||
return self._vad_analyzer
|
return self._vad_analyzer
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the input transport and connect to LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -389,6 +548,11 @@ class LiveKitInputTransport(BaseInputTransport):
|
|||||||
logger.info("LiveKitInputTransport started")
|
logger.info("LiveKitInputTransport started")
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the input transport and disconnect from LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
if self._audio_in_task:
|
if self._audio_in_task:
|
||||||
@@ -396,24 +560,42 @@ class LiveKitInputTransport(BaseInputTransport):
|
|||||||
logger.info("LiveKitInputTransport stopped")
|
logger.info("LiveKitInputTransport stopped")
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the input transport and disconnect from LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
if self._audio_in_task and self._params.audio_in_enabled:
|
if self._audio_in_task and self._params.audio_in_enabled:
|
||||||
await self.cancel_task(self._audio_in_task)
|
await self.cancel_task(self._audio_in_task)
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Setup the input transport with shared client setup.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
await self._client.setup(setup)
|
await self._client.setup(setup)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup input transport and shared resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def push_app_message(self, message: Any, sender: str):
|
async def push_app_message(self, message: Any, sender: str):
|
||||||
|
"""Push an application message as an urgent transport frame.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
message: The message data to send.
|
||||||
|
sender: ID of the message sender.
|
||||||
|
"""
|
||||||
frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender)
|
frame = LiveKitTransportMessageUrgentFrame(message=message, participant_id=sender)
|
||||||
await self.push_frame(frame)
|
await self.push_frame(frame)
|
||||||
|
|
||||||
async def _audio_in_task_handler(self):
|
async def _audio_in_task_handler(self):
|
||||||
|
"""Handle incoming audio frames from participants."""
|
||||||
logger.info("Audio input task started")
|
logger.info("Audio input task started")
|
||||||
audio_iterator = self._client.get_next_audio_frame()
|
audio_iterator = self._client.get_next_audio_frame()
|
||||||
async for audio_data in WatchdogAsyncIterator(audio_iterator, manager=self.task_manager):
|
async for audio_data in WatchdogAsyncIterator(audio_iterator, manager=self.task_manager):
|
||||||
@@ -433,6 +615,7 @@ class LiveKitInputTransport(BaseInputTransport):
|
|||||||
async def _convert_livekit_audio_to_pipecat(
|
async def _convert_livekit_audio_to_pipecat(
|
||||||
self, audio_frame_event: rtc.AudioFrameEvent
|
self, audio_frame_event: rtc.AudioFrameEvent
|
||||||
) -> AudioRawFrame:
|
) -> AudioRawFrame:
|
||||||
|
"""Convert LiveKit audio frame to Pipecat audio frame."""
|
||||||
audio_frame = audio_frame_event.frame
|
audio_frame = audio_frame_event.frame
|
||||||
|
|
||||||
audio_data = await self._resampler.resample(
|
audio_data = await self._resampler.resample(
|
||||||
@@ -447,6 +630,12 @@ class LiveKitInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class LiveKitOutputTransport(BaseOutputTransport):
|
class LiveKitOutputTransport(BaseOutputTransport):
|
||||||
|
"""Handles outgoing media streams and events to LiveKit rooms.
|
||||||
|
|
||||||
|
Manages sending audio frames and data messages to LiveKit room participants,
|
||||||
|
including audio format conversion for LiveKit compatibility.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
transport: BaseTransport,
|
transport: BaseTransport,
|
||||||
@@ -454,6 +643,14 @@ class LiveKitOutputTransport(BaseOutputTransport):
|
|||||||
params: LiveKitParams,
|
params: LiveKitParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the LiveKit output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
transport: The parent transport instance.
|
||||||
|
client: LiveKitTransportClient instance.
|
||||||
|
params: Configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
self._transport = transport
|
self._transport = transport
|
||||||
self._client = client
|
self._client = client
|
||||||
@@ -462,6 +659,11 @@ class LiveKitOutputTransport(BaseOutputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the output transport and connect to LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -475,33 +677,60 @@ class LiveKitOutputTransport(BaseOutputTransport):
|
|||||||
logger.info("LiveKitOutputTransport started")
|
logger.info("LiveKitOutputTransport started")
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the output transport and disconnect from LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
logger.info("LiveKitOutputTransport stopped")
|
logger.info("LiveKitOutputTransport stopped")
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the output transport and disconnect from LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._client.disconnect()
|
await self._client.disconnect()
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Setup the output transport with shared client setup.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
await self._client.setup(setup)
|
await self._client.setup(setup)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup output transport and shared resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._transport.cleanup()
|
await self._transport.cleanup()
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a transport message to participants.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The transport message frame to send.
|
||||||
|
"""
|
||||||
if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)):
|
if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)):
|
||||||
await self._client.send_data(frame.message.encode(), frame.participant_id)
|
await self._client.send_data(frame.message.encode(), frame.participant_id)
|
||||||
else:
|
else:
|
||||||
await self._client.send_data(frame.message.encode())
|
await self._client.send_data(frame.message.encode())
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to write.
|
||||||
|
"""
|
||||||
livekit_audio = self._convert_pipecat_audio_to_livekit(frame.audio)
|
livekit_audio = self._convert_pipecat_audio_to_livekit(frame.audio)
|
||||||
await self._client.publish_audio(livekit_audio)
|
await self._client.publish_audio(livekit_audio)
|
||||||
|
|
||||||
def _convert_pipecat_audio_to_livekit(self, pipecat_audio: bytes) -> rtc.AudioFrame:
|
def _convert_pipecat_audio_to_livekit(self, pipecat_audio: bytes) -> rtc.AudioFrame:
|
||||||
|
"""Convert Pipecat audio data to LiveKit audio frame."""
|
||||||
bytes_per_sample = 2 # Assuming 16-bit audio
|
bytes_per_sample = 2 # Assuming 16-bit audio
|
||||||
total_samples = len(pipecat_audio) // bytes_per_sample
|
total_samples = len(pipecat_audio) // bytes_per_sample
|
||||||
samples_per_channel = total_samples // self._params.audio_out_channels
|
samples_per_channel = total_samples // self._params.audio_out_channels
|
||||||
@@ -515,6 +744,13 @@ class LiveKitOutputTransport(BaseOutputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class LiveKitTransport(BaseTransport):
|
class LiveKitTransport(BaseTransport):
|
||||||
|
"""Transport implementation for LiveKit real-time communication.
|
||||||
|
|
||||||
|
Provides comprehensive LiveKit integration including audio streaming, data
|
||||||
|
messaging, participant management, and room event handling for conversational
|
||||||
|
AI applications.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
url: str,
|
url: str,
|
||||||
@@ -524,6 +760,16 @@ class LiveKitTransport(BaseTransport):
|
|||||||
input_name: Optional[str] = None,
|
input_name: Optional[str] = None,
|
||||||
output_name: Optional[str] = None,
|
output_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the LiveKit transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url: LiveKit server URL to connect to.
|
||||||
|
token: Authentication token for the room.
|
||||||
|
room_name: Name of the LiveKit room to join.
|
||||||
|
params: Configuration parameters for the transport.
|
||||||
|
input_name: Optional name for the input transport.
|
||||||
|
output_name: Optional name for the output transport.
|
||||||
|
"""
|
||||||
super().__init__(input_name=input_name, output_name=output_name)
|
super().__init__(input_name=input_name, output_name=output_name)
|
||||||
|
|
||||||
callbacks = LiveKitCallbacks(
|
callbacks = LiveKitCallbacks(
|
||||||
@@ -556,6 +802,11 @@ class LiveKitTransport(BaseTransport):
|
|||||||
self._register_event_handler("on_call_state_updated")
|
self._register_event_handler("on_call_state_updated")
|
||||||
|
|
||||||
def input(self) -> LiveKitInputTransport:
|
def input(self) -> LiveKitInputTransport:
|
||||||
|
"""Get the input transport for receiving media and events.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The LiveKit input transport instance.
|
||||||
|
"""
|
||||||
if not self._input:
|
if not self._input:
|
||||||
self._input = LiveKitInputTransport(
|
self._input = LiveKitInputTransport(
|
||||||
self, self._client, self._params, name=self._input_name
|
self, self._client, self._params, name=self._input_name
|
||||||
@@ -563,6 +814,11 @@ class LiveKitTransport(BaseTransport):
|
|||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> LiveKitOutputTransport:
|
def output(self) -> LiveKitOutputTransport:
|
||||||
|
"""Get the output transport for sending media and events.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The LiveKit output transport instance.
|
||||||
|
"""
|
||||||
if not self._output:
|
if not self._output:
|
||||||
self._output = LiveKitOutputTransport(
|
self._output = LiveKitOutputTransport(
|
||||||
self, self._client, self._params, name=self._output_name
|
self, self._client, self._params, name=self._output_name
|
||||||
@@ -571,41 +827,84 @@ class LiveKitTransport(BaseTransport):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def participant_id(self) -> str:
|
def participant_id(self) -> str:
|
||||||
|
"""Get the participant ID for this transport.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The participant ID assigned by LiveKit.
|
||||||
|
"""
|
||||||
return self._client.participant_id
|
return self._client.participant_id
|
||||||
|
|
||||||
async def send_audio(self, frame: OutputAudioRawFrame):
|
async def send_audio(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Send an audio frame to the LiveKit room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to send.
|
||||||
|
"""
|
||||||
if self._output:
|
if self._output:
|
||||||
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
await self._output.queue_frame(frame, FrameDirection.DOWNSTREAM)
|
||||||
|
|
||||||
def get_participants(self) -> List[str]:
|
def get_participants(self) -> List[str]:
|
||||||
|
"""Get list of participant IDs in the room.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
List of participant IDs.
|
||||||
|
"""
|
||||||
return self._client.get_participants()
|
return self._client.get_participants()
|
||||||
|
|
||||||
async def get_participant_metadata(self, participant_id: str) -> dict:
|
async def get_participant_metadata(self, participant_id: str) -> dict:
|
||||||
|
"""Get metadata for a specific participant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to get metadata for.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary containing participant metadata.
|
||||||
|
"""
|
||||||
return await self._client.get_participant_metadata(participant_id)
|
return await self._client.get_participant_metadata(participant_id)
|
||||||
|
|
||||||
async def set_metadata(self, metadata: str):
|
async def set_metadata(self, metadata: str):
|
||||||
|
"""Set metadata for the local participant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
metadata: Metadata string to set.
|
||||||
|
"""
|
||||||
await self._client.set_participant_metadata(metadata)
|
await self._client.set_participant_metadata(metadata)
|
||||||
|
|
||||||
async def mute_participant(self, participant_id: str):
|
async def mute_participant(self, participant_id: str):
|
||||||
|
"""Mute a specific participant's audio tracks.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to mute.
|
||||||
|
"""
|
||||||
await self._client.mute_participant(participant_id)
|
await self._client.mute_participant(participant_id)
|
||||||
|
|
||||||
async def unmute_participant(self, participant_id: str):
|
async def unmute_participant(self, participant_id: str):
|
||||||
|
"""Unmute a specific participant's audio tracks.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to unmute.
|
||||||
|
"""
|
||||||
await self._client.unmute_participant(participant_id)
|
await self._client.unmute_participant(participant_id)
|
||||||
|
|
||||||
async def _on_connected(self):
|
async def _on_connected(self):
|
||||||
|
"""Handle room connected events."""
|
||||||
await self._call_event_handler("on_connected")
|
await self._call_event_handler("on_connected")
|
||||||
|
|
||||||
async def _on_disconnected(self):
|
async def _on_disconnected(self):
|
||||||
|
"""Handle room disconnected events."""
|
||||||
await self._call_event_handler("on_disconnected")
|
await self._call_event_handler("on_disconnected")
|
||||||
|
|
||||||
async def _on_participant_connected(self, participant_id: str):
|
async def _on_participant_connected(self, participant_id: str):
|
||||||
|
"""Handle participant connected events."""
|
||||||
await self._call_event_handler("on_participant_connected", participant_id)
|
await self._call_event_handler("on_participant_connected", participant_id)
|
||||||
|
|
||||||
async def _on_participant_disconnected(self, participant_id: str):
|
async def _on_participant_disconnected(self, participant_id: str):
|
||||||
|
"""Handle participant disconnected events."""
|
||||||
await self._call_event_handler("on_participant_disconnected", participant_id)
|
await self._call_event_handler("on_participant_disconnected", participant_id)
|
||||||
await self._call_event_handler("on_participant_left", participant_id, "disconnected")
|
await self._call_event_handler("on_participant_left", participant_id, "disconnected")
|
||||||
|
|
||||||
async def _on_audio_track_subscribed(self, participant_id: str):
|
async def _on_audio_track_subscribed(self, participant_id: str):
|
||||||
|
"""Handle audio track subscribed events."""
|
||||||
await self._call_event_handler("on_audio_track_subscribed", participant_id)
|
await self._call_event_handler("on_audio_track_subscribed", participant_id)
|
||||||
participant = self._client.room.remote_participants.get(participant_id)
|
participant = self._client.room.remote_participants.get(participant_id)
|
||||||
if participant:
|
if participant:
|
||||||
@@ -615,19 +914,33 @@ class LiveKitTransport(BaseTransport):
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def _on_audio_track_unsubscribed(self, participant_id: str):
|
async def _on_audio_track_unsubscribed(self, participant_id: str):
|
||||||
|
"""Handle audio track unsubscribed events."""
|
||||||
await self._call_event_handler("on_audio_track_unsubscribed", participant_id)
|
await self._call_event_handler("on_audio_track_unsubscribed", participant_id)
|
||||||
|
|
||||||
async def _on_data_received(self, data: bytes, participant_id: str):
|
async def _on_data_received(self, data: bytes, participant_id: str):
|
||||||
|
"""Handle data received events."""
|
||||||
if self._input:
|
if self._input:
|
||||||
await self._input.push_app_message(data.decode(), participant_id)
|
await self._input.push_app_message(data.decode(), participant_id)
|
||||||
await self._call_event_handler("on_data_received", data, participant_id)
|
await self._call_event_handler("on_data_received", data, participant_id)
|
||||||
|
|
||||||
async def send_message(self, message: str, participant_id: Optional[str] = None):
|
async def send_message(self, message: str, participant_id: Optional[str] = None):
|
||||||
|
"""Send a message to participants in the room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
message: The message string to send.
|
||||||
|
participant_id: Optional specific participant to send to.
|
||||||
|
"""
|
||||||
if self._output:
|
if self._output:
|
||||||
frame = LiveKitTransportMessageFrame(message=message, participant_id=participant_id)
|
frame = LiveKitTransportMessageFrame(message=message, participant_id=participant_id)
|
||||||
await self._output.send_message(frame)
|
await self._output.send_message(frame)
|
||||||
|
|
||||||
async def send_message_urgent(self, message: str, participant_id: Optional[str] = None):
|
async def send_message_urgent(self, message: str, participant_id: Optional[str] = None):
|
||||||
|
"""Send an urgent message to participants in the room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
message: The urgent message string to send.
|
||||||
|
participant_id: Optional specific participant to send to.
|
||||||
|
"""
|
||||||
if self._output:
|
if self._output:
|
||||||
frame = LiveKitTransportMessageUrgentFrame(
|
frame = LiveKitTransportMessageUrgentFrame(
|
||||||
message=message, participant_id=participant_id
|
message=message, participant_id=participant_id
|
||||||
@@ -635,19 +948,36 @@ class LiveKitTransport(BaseTransport):
|
|||||||
await self._output.send_message(frame)
|
await self._output.send_message(frame)
|
||||||
|
|
||||||
async def on_room_event(self, event):
|
async def on_room_event(self, event):
|
||||||
|
"""Handle room events.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event: The room event to handle.
|
||||||
|
"""
|
||||||
# Handle room events
|
# Handle room events
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def on_participant_event(self, event):
|
async def on_participant_event(self, event):
|
||||||
|
"""Handle participant events.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event: The participant event to handle.
|
||||||
|
"""
|
||||||
# Handle participant events
|
# Handle participant events
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def on_track_event(self, event):
|
async def on_track_event(self, event):
|
||||||
|
"""Handle track events.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
event: The track event to handle.
|
||||||
|
"""
|
||||||
# Handle track events
|
# Handle track events
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def _on_call_state_updated(self, state: str):
|
async def _on_call_state_updated(self, state: str):
|
||||||
|
"""Handle call state update events."""
|
||||||
await self._call_event_handler("on_call_state_updated", self, state)
|
await self._call_event_handler("on_call_state_updated", self, state)
|
||||||
|
|
||||||
async def _on_first_participant_joined(self, participant_id: str):
|
async def _on_first_participant_joined(self, participant_id: str):
|
||||||
|
"""Handle first participant joined events."""
|
||||||
await self._call_event_handler("on_first_participant_joined", participant_id)
|
await self._call_event_handler("on_first_participant_joined", participant_id)
|
||||||
|
|||||||
@@ -1,3 +1,16 @@
|
|||||||
|
#
|
||||||
|
# Copyright (c) 2024–2025, Daily
|
||||||
|
#
|
||||||
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Tavus transport implementation for Pipecat.
|
||||||
|
|
||||||
|
This module provides integration with the Tavus platform for creating conversational
|
||||||
|
AI applications with avatars. It manages conversation sessions and provides real-time
|
||||||
|
audio/video streaming capabilities through the Tavus API.
|
||||||
|
"""
|
||||||
|
|
||||||
import os
|
import os
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any, Awaitable, Callable, Mapping, Optional
|
from typing import Any, Awaitable, Callable, Mapping, Optional
|
||||||
@@ -31,8 +44,10 @@ from pipecat.transports.services.daily import (
|
|||||||
|
|
||||||
|
|
||||||
class TavusApi:
|
class TavusApi:
|
||||||
"""
|
"""Helper class for interacting with the Tavus API (v2).
|
||||||
A helper class for interacting with the Tavus API (v2).
|
|
||||||
|
Provides methods for creating and managing conversations with Tavus avatars,
|
||||||
|
including conversation lifecycle management and persona information retrieval.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
BASE_URL = "https://tavusapi.com/v2"
|
BASE_URL = "https://tavusapi.com/v2"
|
||||||
@@ -40,12 +55,11 @@ class TavusApi:
|
|||||||
MOCK_PERSONA_NAME = "TestTavusTransport"
|
MOCK_PERSONA_NAME = "TestTavusTransport"
|
||||||
|
|
||||||
def __init__(self, api_key: str, session: aiohttp.ClientSession):
|
def __init__(self, api_key: str, session: aiohttp.ClientSession):
|
||||||
"""
|
"""Initialize the TavusApi client.
|
||||||
Initialize the TavusApi client.
|
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
api_key (str): Tavus API key.
|
api_key: Tavus API key for authentication.
|
||||||
session (aiohttp.ClientSession): An aiohttp session for making HTTP requests.
|
session: An aiohttp session for making HTTP requests.
|
||||||
"""
|
"""
|
||||||
self._api_key = api_key
|
self._api_key = api_key
|
||||||
self._session = session
|
self._session = session
|
||||||
@@ -54,6 +68,15 @@ class TavusApi:
|
|||||||
self._dev_room_url = os.getenv("TAVUS_SAMPLE_ROOM_URL")
|
self._dev_room_url = os.getenv("TAVUS_SAMPLE_ROOM_URL")
|
||||||
|
|
||||||
async def create_conversation(self, replica_id: str, persona_id: str) -> dict:
|
async def create_conversation(self, replica_id: str, persona_id: str) -> dict:
|
||||||
|
"""Create a new conversation with the specified replica and persona.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
replica_id: ID of the replica to use in the conversation.
|
||||||
|
persona_id: ID of the persona to use in the conversation.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Dictionary containing conversation_id and conversation_url.
|
||||||
|
"""
|
||||||
if self._dev_room_url:
|
if self._dev_room_url:
|
||||||
return {
|
return {
|
||||||
"conversation_id": self.MOCK_CONVERSATION_ID,
|
"conversation_id": self.MOCK_CONVERSATION_ID,
|
||||||
@@ -73,6 +96,11 @@ class TavusApi:
|
|||||||
return response
|
return response
|
||||||
|
|
||||||
async def end_conversation(self, conversation_id: str):
|
async def end_conversation(self, conversation_id: str):
|
||||||
|
"""End an existing conversation.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
conversation_id: ID of the conversation to end.
|
||||||
|
"""
|
||||||
if conversation_id is None or conversation_id == self.MOCK_CONVERSATION_ID:
|
if conversation_id is None or conversation_id == self.MOCK_CONVERSATION_ID:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -82,6 +110,14 @@ class TavusApi:
|
|||||||
logger.debug(f"Ended Tavus conversation {conversation_id}")
|
logger.debug(f"Ended Tavus conversation {conversation_id}")
|
||||||
|
|
||||||
async def get_persona_name(self, persona_id: str) -> str:
|
async def get_persona_name(self, persona_id: str) -> str:
|
||||||
|
"""Get the name of a persona by ID.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
persona_id: ID of the persona to retrieve.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The name of the persona.
|
||||||
|
"""
|
||||||
if self._dev_room_url is not None:
|
if self._dev_room_url is not None:
|
||||||
return self.MOCK_PERSONA_NAME
|
return self.MOCK_PERSONA_NAME
|
||||||
|
|
||||||
@@ -94,11 +130,11 @@ class TavusApi:
|
|||||||
|
|
||||||
|
|
||||||
class TavusCallbacks(BaseModel):
|
class TavusCallbacks(BaseModel):
|
||||||
"""Callback handlers for the Tavus events.
|
"""Callback handlers for Tavus events.
|
||||||
|
|
||||||
Attributes:
|
Parameters:
|
||||||
on_participant_joined: Called when a participant joins.
|
on_participant_joined: Called when a participant joins the conversation.
|
||||||
on_participant_left: Called when a participant leaves.
|
on_participant_left: Called when a participant leaves the conversation.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
on_participant_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
|
on_participant_joined: Callable[[Mapping[str, Any]], Awaitable[None]]
|
||||||
@@ -106,7 +142,13 @@ class TavusCallbacks(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
class TavusParams(DailyParams):
|
class TavusParams(DailyParams):
|
||||||
"""Configuration parameters for the Tavus transport."""
|
"""Configuration parameters for the Tavus transport.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
audio_in_enabled: Whether to enable audio input from participants.
|
||||||
|
audio_out_enabled: Whether to enable audio output to participants.
|
||||||
|
microphone_out_enabled: Whether to enable microphone output track.
|
||||||
|
"""
|
||||||
|
|
||||||
audio_in_enabled: bool = True
|
audio_in_enabled: bool = True
|
||||||
audio_out_enabled: bool = True
|
audio_out_enabled: bool = True
|
||||||
@@ -114,24 +156,14 @@ class TavusParams(DailyParams):
|
|||||||
|
|
||||||
|
|
||||||
class TavusTransportClient:
|
class TavusTransportClient:
|
||||||
"""
|
"""Transport client that integrates Pipecat with the Tavus platform.
|
||||||
|
|
||||||
A transport client that integrates a Pipecat Bot with the Tavus platform by managing
|
A transport client that integrates a Pipecat Bot with the Tavus platform by managing
|
||||||
conversation sessions using the Tavus API.
|
conversation sessions using the Tavus API.
|
||||||
|
|
||||||
This client uses `TavusApi` to interact with the Tavus backend services. When a conversation
|
This client uses `TavusApi` to interact with the Tavus backend services. When a conversation
|
||||||
is started via `TavusApi`, Tavus provides a `roomURL` that can be used to connect the Pipecat Bot
|
is started via `TavusApi`, Tavus provides a `roomURL` that can be used to connect the Pipecat Bot
|
||||||
into the same virtual room where the TavusBot is operating.
|
into the same virtual room where the TavusBot is operating.
|
||||||
|
|
||||||
Args:
|
|
||||||
bot_name (str): The name of the Pipecat bot instance.
|
|
||||||
params (TavusParams): Optional parameters for Tavus operation. Defaults to `TavusParams()`.
|
|
||||||
callbacks (TavusCallbacks): Callback handlers for Tavus-related events.
|
|
||||||
api_key (str): API key for authenticating with Tavus API.
|
|
||||||
replica_id (str): ID of the replica to use in the Tavus conversation.
|
|
||||||
persona_id (str): ID of the Tavus persona. Defaults to "pipecat-stream", which signals Tavus to use
|
|
||||||
the TTS voice of the Pipecat bot instead of a Tavus persona voice.
|
|
||||||
session (aiohttp.ClientSession): The aiohttp session for making async HTTP requests.
|
|
||||||
sample_rate: Audio sample rate to be used by the client.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -145,6 +177,19 @@ class TavusTransportClient:
|
|||||||
persona_id: str = "pipecat-stream",
|
persona_id: str = "pipecat-stream",
|
||||||
session: aiohttp.ClientSession,
|
session: aiohttp.ClientSession,
|
||||||
) -> None:
|
) -> None:
|
||||||
|
"""Initialize the Tavus transport client.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bot_name: The name of the Pipecat bot instance.
|
||||||
|
params: Optional parameters for Tavus operation.
|
||||||
|
callbacks: Callback handlers for Tavus-related events.
|
||||||
|
api_key: API key for authenticating with Tavus API.
|
||||||
|
replica_id: ID of the replica to use in the Tavus conversation.
|
||||||
|
persona_id: ID of the Tavus persona. Defaults to "pipecat-stream",
|
||||||
|
which signals Tavus to use the TTS voice of the Pipecat bot
|
||||||
|
instead of a Tavus persona voice.
|
||||||
|
session: The aiohttp session for making async HTTP requests.
|
||||||
|
"""
|
||||||
self._bot_name = bot_name
|
self._bot_name = bot_name
|
||||||
self._api = TavusApi(api_key, session)
|
self._api = TavusApi(api_key, session)
|
||||||
self._replica_id = replica_id
|
self._replica_id = replica_id
|
||||||
@@ -155,11 +200,17 @@ class TavusTransportClient:
|
|||||||
self._params = params
|
self._params = params
|
||||||
|
|
||||||
async def _initialize(self) -> str:
|
async def _initialize(self) -> str:
|
||||||
|
"""Initialize the conversation and return the room URL."""
|
||||||
response = await self._api.create_conversation(self._replica_id, self._persona_id)
|
response = await self._api.create_conversation(self._replica_id, self._persona_id)
|
||||||
self._conversation_id = response["conversation_id"]
|
self._conversation_id = response["conversation_id"]
|
||||||
return response["conversation_url"]
|
return response["conversation_url"]
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Setup the client and initialize the conversation.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
if self._conversation_id is not None:
|
if self._conversation_id is not None:
|
||||||
logger.debug(f"Conversation ID already defined: {self._conversation_id}")
|
logger.debug(f"Conversation ID already defined: {self._conversation_id}")
|
||||||
return
|
return
|
||||||
@@ -206,29 +257,44 @@ class TavusTransportClient:
|
|||||||
self._conversation_id = None
|
self._conversation_id = None
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup client resources."""
|
||||||
try:
|
try:
|
||||||
await self._client.cleanup()
|
await self._client.cleanup()
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception(f"Exception during cleanup: {e}")
|
logger.exception(f"Exception during cleanup: {e}")
|
||||||
|
|
||||||
async def _on_joined(self, data):
|
async def _on_joined(self, data):
|
||||||
|
"""Handle joined event."""
|
||||||
logger.debug("TavusTransportClient joined!")
|
logger.debug("TavusTransportClient joined!")
|
||||||
|
|
||||||
async def _on_left(self):
|
async def _on_left(self):
|
||||||
|
"""Handle left event."""
|
||||||
logger.debug("TavusTransportClient left!")
|
logger.debug("TavusTransportClient left!")
|
||||||
|
|
||||||
async def _on_handle_callback(self, event_name, *args, **kwargs):
|
async def _on_handle_callback(self, event_name, *args, **kwargs):
|
||||||
|
"""Handle generic callback events."""
|
||||||
logger.trace(f"[Callback] {event_name} called with args={args}, kwargs={kwargs}")
|
logger.trace(f"[Callback] {event_name} called with args={args}, kwargs={kwargs}")
|
||||||
|
|
||||||
async def get_persona_name(self) -> str:
|
async def get_persona_name(self) -> str:
|
||||||
|
"""Get the persona name from the API.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The name of the current persona.
|
||||||
|
"""
|
||||||
return await self._api.get_persona_name(self._persona_id)
|
return await self._api.get_persona_name(self._persona_id)
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the client and join the room.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
logger.debug("TavusTransportClient start invoked!")
|
logger.debug("TavusTransportClient start invoked!")
|
||||||
await self._client.start(frame)
|
await self._client.start(frame)
|
||||||
await self._client.join()
|
await self._client.join()
|
||||||
|
|
||||||
async def stop(self):
|
async def stop(self):
|
||||||
|
"""Stop the client and end the conversation."""
|
||||||
await self._client.leave()
|
await self._client.leave()
|
||||||
await self._api.end_conversation(self._conversation_id)
|
await self._api.end_conversation(self._conversation_id)
|
||||||
self._conversation_id = None
|
self._conversation_id = None
|
||||||
@@ -241,6 +307,15 @@ class TavusTransportClient:
|
|||||||
video_source: str = "camera",
|
video_source: str = "camera",
|
||||||
color_format: str = "RGB",
|
color_format: str = "RGB",
|
||||||
):
|
):
|
||||||
|
"""Capture video from a participant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to capture video from.
|
||||||
|
callback: Callback function to handle video frames.
|
||||||
|
framerate: Desired framerate for video capture.
|
||||||
|
video_source: Video source to capture from.
|
||||||
|
color_format: Color format for video frames.
|
||||||
|
"""
|
||||||
await self._client.capture_participant_video(
|
await self._client.capture_participant_video(
|
||||||
participant_id, callback, framerate, video_source, color_format
|
participant_id, callback, framerate, video_source, color_format
|
||||||
)
|
)
|
||||||
@@ -253,22 +328,47 @@ class TavusTransportClient:
|
|||||||
sample_rate: int = 16000,
|
sample_rate: int = 16000,
|
||||||
callback_interval_ms: int = 20,
|
callback_interval_ms: int = 20,
|
||||||
):
|
):
|
||||||
|
"""Capture audio from a participant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_id: ID of the participant to capture audio from.
|
||||||
|
callback: Callback function to handle audio data.
|
||||||
|
audio_source: Audio source to capture from.
|
||||||
|
sample_rate: Desired sample rate for audio capture.
|
||||||
|
callback_interval_ms: Interval between audio callbacks in milliseconds.
|
||||||
|
"""
|
||||||
await self._client.capture_participant_audio(
|
await self._client.capture_participant_audio(
|
||||||
participant_id, callback, audio_source, sample_rate, callback_interval_ms
|
participant_id, callback, audio_source, sample_rate, callback_interval_ms
|
||||||
)
|
)
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a message to participants.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The message frame to send.
|
||||||
|
"""
|
||||||
await self._client.send_message(frame)
|
await self._client.send_message(frame)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def out_sample_rate(self) -> int:
|
def out_sample_rate(self) -> int:
|
||||||
|
"""Get the output sample rate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The output sample rate in Hz.
|
||||||
|
"""
|
||||||
return self._client.out_sample_rate
|
return self._client.out_sample_rate
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def in_sample_rate(self) -> int:
|
def in_sample_rate(self) -> int:
|
||||||
|
"""Get the input sample rate.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The input sample rate in Hz.
|
||||||
|
"""
|
||||||
return self._client.in_sample_rate
|
return self._client.in_sample_rate
|
||||||
|
|
||||||
async def send_interrupt_message(self) -> None:
|
async def send_interrupt_message(self) -> None:
|
||||||
|
"""Send an interrupt message to the conversation."""
|
||||||
transport_frame = TransportMessageUrgentFrame(
|
transport_frame = TransportMessageUrgentFrame(
|
||||||
message={
|
message={
|
||||||
"message_type": "conversation",
|
"message_type": "conversation",
|
||||||
@@ -279,6 +379,12 @@ class TavusTransportClient:
|
|||||||
await self.send_message(transport_frame)
|
await self.send_message(transport_frame)
|
||||||
|
|
||||||
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
|
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
|
||||||
|
"""Update subscription settings for participants.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_settings: Per-participant subscription settings.
|
||||||
|
profile_settings: Global subscription profile settings.
|
||||||
|
"""
|
||||||
if not self._client:
|
if not self._client:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -287,11 +393,21 @@ class TavusTransportClient:
|
|||||||
)
|
)
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to write.
|
||||||
|
"""
|
||||||
if not self._client:
|
if not self._client:
|
||||||
return
|
return
|
||||||
await self._client.write_audio_frame(frame)
|
await self._client.write_audio_frame(frame)
|
||||||
|
|
||||||
async def register_audio_destination(self, destination: str):
|
async def register_audio_destination(self, destination: str):
|
||||||
|
"""Register an audio destination for output.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
destination: The destination identifier to register.
|
||||||
|
"""
|
||||||
if not self._client:
|
if not self._client:
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -299,12 +415,25 @@ class TavusTransportClient:
|
|||||||
|
|
||||||
|
|
||||||
class TavusInputTransport(BaseInputTransport):
|
class TavusInputTransport(BaseInputTransport):
|
||||||
|
"""Input transport for receiving audio and events from Tavus conversations.
|
||||||
|
|
||||||
|
Handles incoming audio streams from participants and manages audio capture
|
||||||
|
from the Daily room connected to the Tavus conversation.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
client: TavusTransportClient,
|
client: TavusTransportClient,
|
||||||
params: TransportParams,
|
params: TransportParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the Tavus input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: The Tavus transport client instance.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
self._client = client
|
self._client = client
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -314,14 +443,25 @@ class TavusInputTransport(BaseInputTransport):
|
|||||||
self._initialized = False
|
self._initialized = False
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Setup the input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
await self._client.setup(setup)
|
await self._client.setup(setup)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup input transport resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._client.cleanup()
|
await self._client.cleanup()
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -333,14 +473,29 @@ class TavusInputTransport(BaseInputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._client.stop()
|
await self._client.stop()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the input transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._client.stop()
|
await self._client.stop()
|
||||||
|
|
||||||
async def start_capturing_audio(self, participant):
|
async def start_capturing_audio(self, participant):
|
||||||
|
"""Start capturing audio from a participant.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant: The participant to capture audio from.
|
||||||
|
"""
|
||||||
if self._params.audio_in_enabled:
|
if self._params.audio_in_enabled:
|
||||||
logger.info(
|
logger.info(
|
||||||
f"TavusTransportClient start capturing audio for participant {participant['id']}"
|
f"TavusTransportClient start capturing audio for participant {participant['id']}"
|
||||||
@@ -354,6 +509,7 @@ class TavusInputTransport(BaseInputTransport):
|
|||||||
async def _on_participant_audio_data(
|
async def _on_participant_audio_data(
|
||||||
self, participant_id: str, audio: AudioData, audio_source: str
|
self, participant_id: str, audio: AudioData, audio_source: str
|
||||||
):
|
):
|
||||||
|
"""Handle received participant audio data."""
|
||||||
frame = InputAudioRawFrame(
|
frame = InputAudioRawFrame(
|
||||||
audio=audio.audio_frames,
|
audio=audio.audio_frames,
|
||||||
sample_rate=audio.audio_frames,
|
sample_rate=audio.audio_frames,
|
||||||
@@ -364,12 +520,25 @@ class TavusInputTransport(BaseInputTransport):
|
|||||||
|
|
||||||
|
|
||||||
class TavusOutputTransport(BaseOutputTransport):
|
class TavusOutputTransport(BaseOutputTransport):
|
||||||
|
"""Output transport for sending audio and events to Tavus conversations.
|
||||||
|
|
||||||
|
Handles outgoing audio streams to participants and manages the custom
|
||||||
|
audio track expected by the Tavus platform.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
client: TavusTransportClient,
|
client: TavusTransportClient,
|
||||||
params: TransportParams,
|
params: TransportParams,
|
||||||
**kwargs,
|
**kwargs,
|
||||||
):
|
):
|
||||||
|
"""Initialize the Tavus output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
client: The Tavus transport client instance.
|
||||||
|
params: Transport configuration parameters.
|
||||||
|
**kwargs: Additional arguments passed to parent class.
|
||||||
|
"""
|
||||||
super().__init__(params, **kwargs)
|
super().__init__(params, **kwargs)
|
||||||
self._client = client
|
self._client = client
|
||||||
self._params = params
|
self._params = params
|
||||||
@@ -380,14 +549,25 @@ class TavusOutputTransport(BaseOutputTransport):
|
|||||||
self._transport_destination: Optional[str] = "stream"
|
self._transport_destination: Optional[str] = "stream"
|
||||||
|
|
||||||
async def setup(self, setup: FrameProcessorSetup):
|
async def setup(self, setup: FrameProcessorSetup):
|
||||||
|
"""Setup the output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
setup: The frame processor setup configuration.
|
||||||
|
"""
|
||||||
await super().setup(setup)
|
await super().setup(setup)
|
||||||
await self._client.setup(setup)
|
await self._client.setup(setup)
|
||||||
|
|
||||||
async def cleanup(self):
|
async def cleanup(self):
|
||||||
|
"""Cleanup output transport resources."""
|
||||||
await super().cleanup()
|
await super().cleanup()
|
||||||
await self._client.cleanup()
|
await self._client.cleanup()
|
||||||
|
|
||||||
async def start(self, frame: StartFrame):
|
async def start(self, frame: StartFrame):
|
||||||
|
"""Start the output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The start frame containing initialization parameters.
|
||||||
|
"""
|
||||||
await super().start(frame)
|
await super().start(frame)
|
||||||
|
|
||||||
if self._initialized:
|
if self._initialized:
|
||||||
@@ -403,51 +583,72 @@ class TavusOutputTransport(BaseOutputTransport):
|
|||||||
await self.set_transport_ready(frame)
|
await self.set_transport_ready(frame)
|
||||||
|
|
||||||
async def stop(self, frame: EndFrame):
|
async def stop(self, frame: EndFrame):
|
||||||
|
"""Stop the output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The end frame signaling transport shutdown.
|
||||||
|
"""
|
||||||
await super().stop(frame)
|
await super().stop(frame)
|
||||||
await self._client.stop()
|
await self._client.stop()
|
||||||
|
|
||||||
async def cancel(self, frame: CancelFrame):
|
async def cancel(self, frame: CancelFrame):
|
||||||
|
"""Cancel the output transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The cancel frame signaling immediate cancellation.
|
||||||
|
"""
|
||||||
await super().cancel(frame)
|
await super().cancel(frame)
|
||||||
await self._client.stop()
|
await self._client.stop()
|
||||||
|
|
||||||
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame):
|
||||||
|
"""Send a message to participants.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The message frame to send.
|
||||||
|
"""
|
||||||
logger.info(f"TavusOutputTransport sending message {frame}")
|
logger.info(f"TavusOutputTransport sending message {frame}")
|
||||||
await self._client.send_message(frame)
|
await self._client.send_message(frame)
|
||||||
|
|
||||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||||
|
"""Process frames and handle interruptions.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The frame to process.
|
||||||
|
direction: The direction of frame flow in the pipeline.
|
||||||
|
"""
|
||||||
await super().process_frame(frame, direction)
|
await super().process_frame(frame, direction)
|
||||||
if isinstance(frame, StartInterruptionFrame):
|
if isinstance(frame, StartInterruptionFrame):
|
||||||
await self._handle_interruptions()
|
await self._handle_interruptions()
|
||||||
|
|
||||||
async def _handle_interruptions(self):
|
async def _handle_interruptions(self):
|
||||||
|
"""Handle interruption events by sending interrupt message."""
|
||||||
await self._client.send_interrupt_message()
|
await self._client.send_interrupt_message()
|
||||||
|
|
||||||
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
async def write_audio_frame(self, frame: OutputAudioRawFrame):
|
||||||
|
"""Write an audio frame to the Tavus transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
frame: The audio frame to write.
|
||||||
|
"""
|
||||||
# This is the custom track destination expected by Tavus
|
# This is the custom track destination expected by Tavus
|
||||||
frame.transport_destination = self._transport_destination
|
frame.transport_destination = self._transport_destination
|
||||||
await self._client.write_audio_frame(frame)
|
await self._client.write_audio_frame(frame)
|
||||||
|
|
||||||
async def register_audio_destination(self, destination: str):
|
async def register_audio_destination(self, destination: str):
|
||||||
|
"""Register an audio destination.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
destination: The destination identifier to register.
|
||||||
|
"""
|
||||||
await self._client.register_audio_destination(destination)
|
await self._client.register_audio_destination(destination)
|
||||||
|
|
||||||
|
|
||||||
class TavusTransport(BaseTransport):
|
class TavusTransport(BaseTransport):
|
||||||
"""
|
"""Transport implementation for Tavus video calls.
|
||||||
Transport implementation for Tavus video calls.
|
|
||||||
|
|
||||||
When used, the Pipecat bot joins the same virtual room as the Tavus Avatar and the user.
|
When used, the Pipecat bot joins the same virtual room as the Tavus Avatar and the user.
|
||||||
This is achieved by using `TavusTransportClient`, which initiates the conversation via
|
This is achieved by using `TavusTransportClient`, which initiates the conversation via
|
||||||
`TavusApi` and obtains a room URL that all participants connect to.
|
`TavusApi` and obtains a room URL that all participants connect to.
|
||||||
|
|
||||||
Args:
|
|
||||||
bot_name (str): The name of the Pipecat bot.
|
|
||||||
session (aiohttp.ClientSession): aiohttp session used for async HTTP requests.
|
|
||||||
api_key (str): Tavus API key for authentication.
|
|
||||||
replica_id (str): ID of the replica model used for voice generation.
|
|
||||||
persona_id (str): ID of the Tavus persona. Defaults to "pipecat-stream" to use the Pipecat TTS voice.
|
|
||||||
params (TavusParams): Optional Tavus-specific configuration parameters.
|
|
||||||
input_name (Optional[str]): Optional name for the input transport.
|
|
||||||
output_name (Optional[str]): Optional name for the output transport.
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -461,6 +662,19 @@ class TavusTransport(BaseTransport):
|
|||||||
input_name: Optional[str] = None,
|
input_name: Optional[str] = None,
|
||||||
output_name: Optional[str] = None,
|
output_name: Optional[str] = None,
|
||||||
):
|
):
|
||||||
|
"""Initialize the Tavus transport.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
bot_name: The name of the Pipecat bot.
|
||||||
|
session: aiohttp session used for async HTTP requests.
|
||||||
|
api_key: Tavus API key for authentication.
|
||||||
|
replica_id: ID of the replica model used for voice generation.
|
||||||
|
persona_id: ID of the Tavus persona. Defaults to "pipecat-stream"
|
||||||
|
to use the Pipecat TTS voice.
|
||||||
|
params: Optional Tavus-specific configuration parameters.
|
||||||
|
input_name: Optional name for the input transport.
|
||||||
|
output_name: Optional name for the output transport.
|
||||||
|
"""
|
||||||
super().__init__(input_name=input_name, output_name=output_name)
|
super().__init__(input_name=input_name, output_name=output_name)
|
||||||
self._params = params
|
self._params = params
|
||||||
|
|
||||||
@@ -487,11 +701,13 @@ class TavusTransport(BaseTransport):
|
|||||||
self._register_event_handler("on_client_disconnected")
|
self._register_event_handler("on_client_disconnected")
|
||||||
|
|
||||||
async def _on_participant_left(self, participant, reason):
|
async def _on_participant_left(self, participant, reason):
|
||||||
|
"""Handle participant left events."""
|
||||||
persona_name = await self._client.get_persona_name()
|
persona_name = await self._client.get_persona_name()
|
||||||
if participant.get("info", {}).get("userName", "") != persona_name:
|
if participant.get("info", {}).get("userName", "") != persona_name:
|
||||||
await self._on_client_disconnected(participant)
|
await self._on_client_disconnected(participant)
|
||||||
|
|
||||||
async def _on_participant_joined(self, participant):
|
async def _on_participant_joined(self, participant):
|
||||||
|
"""Handle participant joined events."""
|
||||||
# get persona, look up persona_name, set this as the bot name to ignore
|
# get persona, look up persona_name, set this as the bot name to ignore
|
||||||
persona_name = await self._client.get_persona_name()
|
persona_name = await self._client.get_persona_name()
|
||||||
|
|
||||||
@@ -513,23 +729,41 @@ class TavusTransport(BaseTransport):
|
|||||||
await self._input.start_capturing_audio(participant)
|
await self._input.start_capturing_audio(participant)
|
||||||
|
|
||||||
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
|
async def update_subscriptions(self, participant_settings=None, profile_settings=None):
|
||||||
|
"""Update subscription settings for participants.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
participant_settings: Per-participant subscription settings.
|
||||||
|
profile_settings: Global subscription profile settings.
|
||||||
|
"""
|
||||||
await self._client.update_subscriptions(
|
await self._client.update_subscriptions(
|
||||||
participant_settings=participant_settings,
|
participant_settings=participant_settings,
|
||||||
profile_settings=profile_settings,
|
profile_settings=profile_settings,
|
||||||
)
|
)
|
||||||
|
|
||||||
def input(self) -> FrameProcessor:
|
def input(self) -> FrameProcessor:
|
||||||
|
"""Get the input transport for receiving media and events.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The Tavus input transport instance.
|
||||||
|
"""
|
||||||
if not self._input:
|
if not self._input:
|
||||||
self._input = TavusInputTransport(client=self._client, params=self._params)
|
self._input = TavusInputTransport(client=self._client, params=self._params)
|
||||||
return self._input
|
return self._input
|
||||||
|
|
||||||
def output(self) -> FrameProcessor:
|
def output(self) -> FrameProcessor:
|
||||||
|
"""Get the output transport for sending media and events.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The Tavus output transport instance.
|
||||||
|
"""
|
||||||
if not self._output:
|
if not self._output:
|
||||||
self._output = TavusOutputTransport(client=self._client, params=self._params)
|
self._output = TavusOutputTransport(client=self._client, params=self._params)
|
||||||
return self._output
|
return self._output
|
||||||
|
|
||||||
async def _on_client_connected(self, participant: Any):
|
async def _on_client_connected(self, participant: Any):
|
||||||
|
"""Handle client connected events."""
|
||||||
await self._call_event_handler("on_client_connected", participant)
|
await self._call_event_handler("on_client_connected", participant)
|
||||||
|
|
||||||
async def _on_client_disconnected(self, participant: Any):
|
async def _on_client_disconnected(self, participant: Any):
|
||||||
|
"""Handle client disconnected events."""
|
||||||
await self._call_event_handler("on_client_disconnected", participant)
|
await self._call_event_handler("on_client_disconnected", participant)
|
||||||
|
|||||||
@@ -4,6 +4,14 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Asyncio task management with watchdog monitoring capabilities.
|
||||||
|
|
||||||
|
This module provides task management functionality with optional watchdog timers
|
||||||
|
to monitor task execution and prevent hanging operations. Includes both abstract
|
||||||
|
base classes and concrete implementations for managing asyncio tasks with
|
||||||
|
comprehensive monitoring and cleanup capabilities.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import time
|
import time
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
@@ -17,6 +25,15 @@ WATCHDOG_TIMEOUT = 5.0
|
|||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class TaskManagerParams:
|
class TaskManagerParams:
|
||||||
|
"""Configuration parameters for task manager initialization.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
loop: The asyncio event loop to use for task management.
|
||||||
|
enable_watchdog_timers: Whether to enable watchdog timers for tasks.
|
||||||
|
enable_watchdog_logging: Whether to log watchdog timing information.
|
||||||
|
watchdog_timeout: Default timeout in seconds for watchdog timers.
|
||||||
|
"""
|
||||||
|
|
||||||
loop: asyncio.AbstractEventLoop
|
loop: asyncio.AbstractEventLoop
|
||||||
enable_watchdog_timers: bool = False
|
enable_watchdog_timers: bool = False
|
||||||
enable_watchdog_logging: bool = False
|
enable_watchdog_logging: bool = False
|
||||||
@@ -24,12 +41,28 @@ class TaskManagerParams:
|
|||||||
|
|
||||||
|
|
||||||
class BaseTaskManager(ABC):
|
class BaseTaskManager(ABC):
|
||||||
|
"""Abstract base class for asyncio task management with watchdog support.
|
||||||
|
|
||||||
|
Provides the interface for creating, monitoring, and managing asyncio tasks
|
||||||
|
with optional watchdog timer functionality to detect stalled operations.
|
||||||
|
"""
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def setup(self, params: TaskManagerParams):
|
def setup(self, params: TaskManagerParams):
|
||||||
|
"""Initialize the task manager with configuration parameters.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: Configuration parameters for task management.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def get_event_loop(self) -> asyncio.AbstractEventLoop:
|
def get_event_loop(self) -> asyncio.AbstractEventLoop:
|
||||||
|
"""Get the event loop used by this task manager.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The asyncio event loop instance.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
@@ -42,21 +75,19 @@ class BaseTaskManager(ABC):
|
|||||||
enable_watchdog_timers: Optional[bool] = None,
|
enable_watchdog_timers: Optional[bool] = None,
|
||||||
watchdog_timeout: Optional[float] = None,
|
watchdog_timeout: Optional[float] = None,
|
||||||
) -> asyncio.Task:
|
) -> asyncio.Task:
|
||||||
"""
|
"""Creates and schedules a new asyncio Task that runs the given coroutine.
|
||||||
Creates and schedules a new asyncio Task that runs the given coroutine.
|
|
||||||
|
|
||||||
The task is added to a global set of created tasks.
|
The task is added to a global set of created tasks.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
loop (asyncio.AbstractEventLoop): The event loop to use for creating the task.
|
coroutine: The coroutine to be executed within the task.
|
||||||
coroutine (Coroutine): The coroutine to be executed within the task.
|
name: The name to assign to the task for identification.
|
||||||
name (str): The name to assign to the task for identification.
|
enable_watchdog_logging: Whether this task should log watchdog processing times.
|
||||||
enable_watchdog_logging(bool): whether this task should log watchdog processing times.
|
enable_watchdog_timers: Whether this task should have a watchdog timer.
|
||||||
enable_watchdog_timers(bool): whether this task should have a watchdog timer.
|
watchdog_timeout: Watchdog timer timeout for this task.
|
||||||
watchdog_timeout(float): watchdog timer timeout for this task.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
asyncio.Task: The created task object.
|
The created task object.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -69,50 +100,67 @@ class BaseTaskManager(ABC):
|
|||||||
is removed from the set of registered tasks upon completion or failure.
|
is removed from the set of registered tasks upon completion or failure.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
task (asyncio.Task): The asyncio Task to wait for.
|
task: The asyncio Task to wait for.
|
||||||
timeout (Optional[float], optional): The maximum number of seconds
|
timeout: The maximum number of seconds to wait for the task to complete.
|
||||||
to wait for the task to complete. If None, waits indefinitely.
|
If None, waits indefinitely.
|
||||||
Defaults to None.
|
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
|
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
|
||||||
"""Cancels the given asyncio Task and awaits its completion with an
|
"""Cancels the given asyncio Task and awaits its completion with an optional timeout.
|
||||||
optional timeout.
|
|
||||||
|
|
||||||
This function removes the task from the set of registered tasks upon
|
This function removes the task from the set of registered tasks upon
|
||||||
completion or failure.
|
completion or failure.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
task (asyncio.Task): The task to be cancelled.
|
task: The task to be cancelled.
|
||||||
timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel.
|
timeout: The optional timeout in seconds to wait for the task to cancel.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def current_tasks(self) -> Sequence[asyncio.Task]:
|
def current_tasks(self) -> Sequence[asyncio.Task]:
|
||||||
"""Returns the list of currently created/registered tasks."""
|
"""Returns the list of currently created/registered tasks.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Sequence of currently managed asyncio tasks.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def task_reset_watchdog(self):
|
def task_reset_watchdog(self):
|
||||||
"""Resets the running task watchdog timer. If not reset, a warning will
|
"""Task reset watchdog timer.
|
||||||
be logged indicating the task is stalling.
|
|
||||||
|
|
||||||
|
Resets the running task watchdog timer. If not reset, a warning will
|
||||||
|
be logged indicating the task is stalling.
|
||||||
"""
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def task_watchdog_enabled(self) -> bool:
|
def task_watchdog_enabled(self) -> bool:
|
||||||
"""Whether the current running task has a watchdog timer enabled."""
|
"""Whether the current running task has a watchdog timer enabled.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the current task has watchdog monitoring active.
|
||||||
|
"""
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class TaskData:
|
class TaskData:
|
||||||
|
"""Internal data structure for tracking task metadata and watchdog state.
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
task: The asyncio Task being managed.
|
||||||
|
watchdog_timer: Event used to reset the watchdog timer.
|
||||||
|
enable_watchdog_logging: Whether to log watchdog timing information.
|
||||||
|
enable_watchdog_timers: Whether watchdog timers are enabled for this task.
|
||||||
|
watchdog_timeout: Timeout in seconds for watchdog warnings.
|
||||||
|
watchdog_task: Optional background task monitoring the watchdog timer.
|
||||||
|
"""
|
||||||
|
|
||||||
task: asyncio.Task
|
task: asyncio.Task
|
||||||
watchdog_timer: asyncio.Event
|
watchdog_timer: asyncio.Event
|
||||||
enable_watchdog_logging: bool
|
enable_watchdog_logging: bool
|
||||||
@@ -122,15 +170,36 @@ class TaskData:
|
|||||||
|
|
||||||
|
|
||||||
class TaskManager(BaseTaskManager):
|
class TaskManager(BaseTaskManager):
|
||||||
|
"""Concrete implementation of BaseTaskManager with full watchdog support.
|
||||||
|
|
||||||
|
Manages asyncio tasks with optional watchdog monitoring to detect stalled
|
||||||
|
operations. Provides comprehensive task lifecycle management including
|
||||||
|
creation, monitoring, cancellation, and cleanup.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
|
"""Initialize the task manager with empty task registry."""
|
||||||
self._tasks: Dict[str, TaskData] = {}
|
self._tasks: Dict[str, TaskData] = {}
|
||||||
self._params: Optional[TaskManagerParams] = None
|
self._params: Optional[TaskManagerParams] = None
|
||||||
|
|
||||||
def setup(self, params: TaskManagerParams):
|
def setup(self, params: TaskManagerParams):
|
||||||
|
"""Initialize the task manager with configuration parameters.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
params: Configuration parameters for task management.
|
||||||
|
"""
|
||||||
if not self._params:
|
if not self._params:
|
||||||
self._params = params
|
self._params = params
|
||||||
|
|
||||||
def get_event_loop(self) -> asyncio.AbstractEventLoop:
|
def get_event_loop(self) -> asyncio.AbstractEventLoop:
|
||||||
|
"""Get the event loop used by this task manager.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The asyncio event loop instance.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If the task manager is not properly set up.
|
||||||
|
"""
|
||||||
if not self._params:
|
if not self._params:
|
||||||
raise Exception("TaskManager is not setup: unable to get event loop")
|
raise Exception("TaskManager is not setup: unable to get event loop")
|
||||||
return self._params.loop
|
return self._params.loop
|
||||||
@@ -144,21 +213,22 @@ class TaskManager(BaseTaskManager):
|
|||||||
enable_watchdog_timers: Optional[bool] = None,
|
enable_watchdog_timers: Optional[bool] = None,
|
||||||
watchdog_timeout: Optional[float] = None,
|
watchdog_timeout: Optional[float] = None,
|
||||||
) -> asyncio.Task:
|
) -> asyncio.Task:
|
||||||
"""
|
"""Creates and schedules a new asyncio Task that runs the given coroutine.
|
||||||
Creates and schedules a new asyncio Task that runs the given coroutine.
|
|
||||||
|
|
||||||
The task is added to a global set of created tasks.
|
The task is added to a global set of created tasks.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
loop (asyncio.AbstractEventLoop): The event loop to use for creating the task.
|
coroutine: The coroutine to be executed within the task.
|
||||||
coroutine (Coroutine): The coroutine to be executed within the task.
|
name: The name to assign to the task for identification.
|
||||||
name (str): The name to assign to the task for identification.
|
enable_watchdog_logging: Whether this task should log watchdog processing time.
|
||||||
enable_watchdog_logging(bool): whether this task should log watchdog processing time.
|
enable_watchdog_timers: Whether this task should have a watchdog timer.
|
||||||
enable_watchdog_timers(bool): whether this task should have a watchdog timer.
|
watchdog_timeout: Watchdog timer timeout for this task.
|
||||||
watchdog_timeout(float): watchdog timer timeout for this task.
|
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
asyncio.Task: The created task object.
|
The created task object.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
Exception: If the task manager is not properly set up.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def run_coroutine():
|
async def run_coroutine():
|
||||||
@@ -208,10 +278,9 @@ class TaskManager(BaseTaskManager):
|
|||||||
is removed from the set of registered tasks upon completion or failure.
|
is removed from the set of registered tasks upon completion or failure.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
task (asyncio.Task): The asyncio Task to wait for.
|
task: The asyncio Task to wait for.
|
||||||
timeout (Optional[float], optional): The maximum number of seconds
|
timeout: The maximum number of seconds to wait for the task to complete.
|
||||||
to wait for the task to complete. If None, waits indefinitely.
|
If None, waits indefinitely.
|
||||||
Defaults to None.
|
|
||||||
"""
|
"""
|
||||||
name = task.get_name()
|
name = task.get_name()
|
||||||
try:
|
try:
|
||||||
@@ -228,16 +297,14 @@ class TaskManager(BaseTaskManager):
|
|||||||
logger.exception(f"{name}: unexpected exception while stopping task: {e}")
|
logger.exception(f"{name}: unexpected exception while stopping task: {e}")
|
||||||
|
|
||||||
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
|
async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None):
|
||||||
"""Cancels the given asyncio Task and awaits its completion with an
|
"""Cancels the given asyncio Task and awaits its completion with an optional timeout.
|
||||||
optional timeout.
|
|
||||||
|
|
||||||
This function removes the task from the set of registered tasks upon
|
This function removes the task from the set of registered tasks upon
|
||||||
completion or failure.
|
completion or failure.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
task (asyncio.Task): The task to be cancelled.
|
task: The task to be cancelled.
|
||||||
timeout (Optional[float]): The optional timeout in seconds to wait for the task to cancel.
|
timeout: The optional timeout in seconds to wait for the task to cancel.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
name = task.get_name()
|
name = task.get_name()
|
||||||
task.cancel()
|
task.cancel()
|
||||||
@@ -260,18 +327,28 @@ class TaskManager(BaseTaskManager):
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
def reset_watchdog(self, task: asyncio.Task):
|
def reset_watchdog(self, task: asyncio.Task):
|
||||||
|
"""Reset the watchdog timer for a specific task.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task: The task whose watchdog timer should be reset.
|
||||||
|
"""
|
||||||
name = task.get_name()
|
name = task.get_name()
|
||||||
if name in self._tasks and self._tasks[name].enable_watchdog_timers:
|
if name in self._tasks and self._tasks[name].enable_watchdog_timers:
|
||||||
self._tasks[name].watchdog_timer.set()
|
self._tasks[name].watchdog_timer.set()
|
||||||
|
|
||||||
def current_tasks(self) -> Sequence[asyncio.Task]:
|
def current_tasks(self) -> Sequence[asyncio.Task]:
|
||||||
"""Returns the list of currently created/registered tasks."""
|
"""Returns the list of currently created/registered tasks.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Sequence of currently managed asyncio tasks.
|
||||||
|
"""
|
||||||
return [data.task for data in self._tasks.values()]
|
return [data.task for data in self._tasks.values()]
|
||||||
|
|
||||||
def task_reset_watchdog(self):
|
def task_reset_watchdog(self):
|
||||||
"""Resets the running task watchdog timer. If not reset on time, a warning
|
"""Task reset watchdog timer.
|
||||||
will be logged indicating the task is stalling.
|
|
||||||
|
|
||||||
|
Resets the running task watchdog timer. If not reset on time, a warning
|
||||||
|
will be logged indicating the task is stalling.
|
||||||
"""
|
"""
|
||||||
task = asyncio.current_task()
|
task = asyncio.current_task()
|
||||||
if task:
|
if task:
|
||||||
@@ -279,6 +356,11 @@ class TaskManager(BaseTaskManager):
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def task_watchdog_enabled(self) -> bool:
|
def task_watchdog_enabled(self) -> bool:
|
||||||
|
"""Whether the current running task has a watchdog timer enabled.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
True if the current task has watchdog monitoring active.
|
||||||
|
"""
|
||||||
task = asyncio.current_task()
|
task = asyncio.current_task()
|
||||||
if not task:
|
if not task:
|
||||||
return False
|
return False
|
||||||
@@ -286,6 +368,11 @@ class TaskManager(BaseTaskManager):
|
|||||||
return name in self._tasks and self._tasks[name].enable_watchdog_timers
|
return name in self._tasks and self._tasks[name].enable_watchdog_timers
|
||||||
|
|
||||||
def _add_task(self, task_data: TaskData):
|
def _add_task(self, task_data: TaskData):
|
||||||
|
"""Add a task to the internal registry and start watchdog if enabled.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task_data: The task data containing task and watchdog configuration.
|
||||||
|
"""
|
||||||
name = task_data.task.get_name()
|
name = task_data.task.get_name()
|
||||||
self._tasks[name] = task_data
|
self._tasks[name] = task_data
|
||||||
if self._params and task_data.enable_watchdog_timers:
|
if self._params and task_data.enable_watchdog_timers:
|
||||||
@@ -295,6 +382,11 @@ class TaskManager(BaseTaskManager):
|
|||||||
task_data.watchdog_task = watchdog_task
|
task_data.watchdog_task = watchdog_task
|
||||||
|
|
||||||
async def _watchdog_task_handler(self, task_data: TaskData):
|
async def _watchdog_task_handler(self, task_data: TaskData):
|
||||||
|
"""Background task that monitors watchdog timer for a specific task.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task_data: The task data containing watchdog configuration.
|
||||||
|
"""
|
||||||
name = task_data.task.get_name()
|
name = task_data.task.get_name()
|
||||||
timer = task_data.watchdog_timer
|
timer = task_data.watchdog_timer
|
||||||
enable_watchdog_logging = task_data.enable_watchdog_logging
|
enable_watchdog_logging = task_data.enable_watchdog_logging
|
||||||
@@ -315,6 +407,11 @@ class TaskManager(BaseTaskManager):
|
|||||||
timer.clear()
|
timer.clear()
|
||||||
|
|
||||||
def _task_done_handler(self, task: asyncio.Task):
|
def _task_done_handler(self, task: asyncio.Task):
|
||||||
|
"""Handle task completion by cleaning up watchdog and removing from registry.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
task: The completed asyncio task.
|
||||||
|
"""
|
||||||
name = task.get_name()
|
name = task.get_name()
|
||||||
try:
|
try:
|
||||||
task_data = self._tasks[name]
|
task_data = self._tasks[name]
|
||||||
|
|||||||
@@ -4,6 +4,13 @@
|
|||||||
# SPDX-License-Identifier: BSD 2-Clause License
|
# SPDX-License-Identifier: BSD 2-Clause License
|
||||||
#
|
#
|
||||||
|
|
||||||
|
"""Watchdog-enabled async iterator wrapper for task monitoring.
|
||||||
|
|
||||||
|
This module provides an async iterator wrapper that automatically resets
|
||||||
|
watchdog timers while waiting for iterator items, preventing false positive
|
||||||
|
watchdog timeouts during legitimate waiting periods.
|
||||||
|
"""
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
from typing import AsyncIterator, Optional
|
from typing import AsyncIterator, Optional
|
||||||
|
|
||||||
@@ -11,10 +18,11 @@ from pipecat.utils.asyncio.task_manager import BaseTaskManager
|
|||||||
|
|
||||||
|
|
||||||
class WatchdogAsyncIterator:
|
class WatchdogAsyncIterator:
|
||||||
"""An asynchronous iterator that monitors activity and resets the current
|
"""Watchdog async iterator wrapper.
|
||||||
|
|
||||||
|
An asynchronous iterator that monitors activity and resets the current
|
||||||
task watchdog timer. This is necessary to avoid task watchdog timers to
|
task watchdog timer. This is necessary to avoid task watchdog timers to
|
||||||
expire while we are waiting to get an item from the iterator.
|
expire while we are waiting to get an item from the iterator.
|
||||||
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -24,6 +32,13 @@ class WatchdogAsyncIterator:
|
|||||||
manager: BaseTaskManager,
|
manager: BaseTaskManager,
|
||||||
timeout: float = 2.0,
|
timeout: float = 2.0,
|
||||||
):
|
):
|
||||||
|
"""Initialize the watchdog async iterator.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
async_iterable: The async iterable to wrap with watchdog monitoring.
|
||||||
|
manager: The task manager for watchdog timer control.
|
||||||
|
timeout: Timeout in seconds between watchdog resets while waiting.
|
||||||
|
"""
|
||||||
self._async_iterable = async_iterable
|
self._async_iterable = async_iterable
|
||||||
self._manager = manager
|
self._manager = manager
|
||||||
self._timeout = timeout
|
self._timeout = timeout
|
||||||
@@ -31,9 +46,22 @@ class WatchdogAsyncIterator:
|
|||||||
self._current_anext_task: Optional[asyncio.Task] = None
|
self._current_anext_task: Optional[asyncio.Task] = None
|
||||||
|
|
||||||
def __aiter__(self):
|
def __aiter__(self):
|
||||||
|
"""Return self as the async iterator.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
This iterator instance.
|
||||||
|
"""
|
||||||
return self
|
return self
|
||||||
|
|
||||||
async def __anext__(self):
|
async def __anext__(self):
|
||||||
|
"""Get the next item from the iterator with watchdog monitoring.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
The next item from the wrapped async iterator.
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
StopAsyncIteration: When the iterator is exhausted.
|
||||||
|
"""
|
||||||
if not self._iter:
|
if not self._iter:
|
||||||
self._iter = await self._ensure_async_iterator(self._async_iterable)
|
self._iter = await self._ensure_async_iterator(self._async_iterable)
|
||||||
|
|
||||||
@@ -43,6 +71,7 @@ class WatchdogAsyncIterator:
|
|||||||
return await self._iter.__anext__()
|
return await self._iter.__anext__()
|
||||||
|
|
||||||
async def _watchdog_anext(self):
|
async def _watchdog_anext(self):
|
||||||
|
"""Get next item while periodically resetting watchdog timer."""
|
||||||
while True:
|
while True:
|
||||||
try:
|
try:
|
||||||
if not self._current_anext_task:
|
if not self._current_anext_task:
|
||||||
@@ -67,6 +96,7 @@ class WatchdogAsyncIterator:
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
async def _ensure_async_iterator(self, obj) -> AsyncIterator:
|
async def _ensure_async_iterator(self, obj) -> AsyncIterator:
|
||||||
|
"""Ensure the object is an async iterator, awaiting if necessary."""
|
||||||
aiter = obj.__aiter__()
|
aiter = obj.__aiter__()
|
||||||
if asyncio.iscoroutine(aiter):
|
if asyncio.iscoroutine(aiter):
|
||||||
aiter = await aiter
|
aiter = await aiter
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user