diff --git a/CHANGELOG.md b/CHANGELOG.md index f153dda13..4d6236f7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,25 +9,48 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `watchdog_coroutine()`. This is a watchdog helper for couroutines. So, + if you have a coroutine that is waiting for a result and that takes a long + time, you will need to wrap it with `watchdog_coroutine()` so the watchdog + timers are reset regularly. + +### Fixed + +- Fixed a `AWSNovaSonicLLMService` issue introduced in 0.0.72. + +## [0.0.73] - 2025-06-26 + +### Fixed + +- Fixed an issue introduced in 0.0.72 that would cause `ElevenLabsTTSService`, + `GladiaSTTService`, `NeuphonicTTSService` and `OpenAIRealtimeBetaLLMService` + to throw an error. + +## [0.0.72] - 2025-06-26 + +### Added + - Added logging and improved error handling to help diagnose and prevent potential Pipeline freezes. +- Added `WatchdogQueue`, `WatchdogPriorityQueue`, `WatchdogEvent` and + `WatchdogAsyncIterator`. These helper utilities reset watchdog timers + appropriately before they expire. When watchdog timers are disabled, the + utilities behave as standard counterparts without side effects. + - Introduce task watchdog timers. Watchdog timers are used to detect if a - Pipecat task is taking longer than expected (by default 5 seconds). It is + Pipecat task is taking longer than expected (by default 5 seconds). Watchdog + timers are disabled by default and can be enabled globally by passing + `enable_watchdog_timers` argument to `PipelineTask` constructor. It is possible to change the default watchdog timer timeout by using the - `watchdog_timeout` constructor argument when creating a `PipelineTask`. With - watchdog timers it is also possible to log how long each processing step is - taking (e.g. processing an element from a queue inside a task). This is done - with the `enable_watchdog_logging` constructor argument when creating a - `PipelineTask.` It is also possible to control these two values per each frame - processor. That is, you can set set `enable_watchdog_logging` and + `watchdog_timeout` argument. You can also log how long it takes to reset the + watchdog timers which is done with the `enable_watchdog_logging`. You can + control all these settings per each frame processor or even per task. That is, + you can set `enable_watchdog_timers`, `enable_watchdog_logging` and `watchdog_timeout` when creating any frame processor through their constructor - arguments. Finally, you can also set these values per task. So, if you are - writing a frame processor that creates multiple tasks and you only want to - enable logging for one of them, you can do so by passing the same argument - names to the `FrameProcessor.create_task()` function. Note that watchdog - timers only work with Pipecat tasks but not if you use `asycio.create_task()` - or similar. + arguments or when you create a task with `FrameProcessor.create_task()`. Note + that watchdog timers only work with Pipecat tasks and will not work if you use + `asycio.create_task()` or similar. - Added `lexicon_names` parameter to `AWSPollyTTSService.InputParams`. @@ -84,6 +107,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue that would cause heartbeat frames to be sent before processors + were started. + - Fixed an event loop blocking issue when using `SentryMetrics`. - Fixed an issue in `FastAPIWebsocketClient` to ensure proper disconnection diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8fee75bd0..1e0594d64 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,36 +41,107 @@ We use Ruff for code linting and formatting. Please ensure your code passes all We follow Google-style docstrings with these specific conventions: -- Class docstrings should fully document all parameters used in `__init__` -- We don't require separate docstrings for `__init__` methods when parameters are documented in the class docstring -- Property methods should have docstrings explaining their purpose and return value +**Regular Classes:** -Example of correctly documented class: +- Class docstring describes the class purpose and key functionality +- `__init__` method has its own docstring with complete `Args:` section documenting all parameters +- All public methods must have docstrings with `Args:` and `Returns:` sections as appropriate + +**Dataclasses:** + +- Class docstring describes the purpose and documents all fields in a `Parameters:` section +- No `__init__` docstring (auto-generated) + +**Properties:** + +- Must have docstrings with `Returns:` section + +**Abstract Methods:** + +- Must have docstrings explaining what subclasses should implement + +**`__init__.py` Files:** + +- **Skip docstrings** for pure import/re-export modules +- **Add brief docstrings** for top-level packages or those with initialization logic + +**Enums:** + +- Class docstring describes the enumeration purpose +- Use `Parameters:` section to document each enum value and its meaning +- No `__init__` docstring (Enums don't have custom constructors) + +#### Examples: ```python -class MyClass: - """Class description. +# Regular class +class MyService(BaseService): + """Description of what the service does. - Additional details about the class. - - Args: - param1: Description of first parameter. - param2: Description of second parameter. + Provides detailed explanation of the service's functionality, + key features, and usage patterns. """ - def __init__(self, param1, param2): - # No docstring required here as parameters are documented above - self.param1 = param1 - self.param2 = param2 + def __init__(self, param1: str, param2: bool = True, **kwargs): + """Initialize the service. + + Args: + param1: Description of param1. + param2: Description of param2. Defaults to True. + **kwargs: Additional arguments passed to parent. + """ + super().__init__(**kwargs) @property - def some_property(self) -> str: - """Get the formatted property value. + def sample_rate(self) -> int: + """Get the current sample rate. Returns: - A string representation of the property. + The sample rate in Hz. """ - return f"Property: {self.param1}" + return self._sample_rate + + async def process_data(self, data: str) -> bool: + """Process the provided data. + + Args: + data: The data to process. + + Returns: + True if processing succeeded. + """ + pass + +# Dataclass +@dataclass +class ConfigParams: + """Configuration parameters for the service. + + Parameters: + host: The host address. + port: The port number. Defaults to 8080. + timeout: Connection timeout in seconds. + """ + + host: str + port: int = 8080 + timeout: float = 30.0 + +# Enum class +class Status(Enum): + """Status codes for processing operations. + + Parameters: + PENDING: Operation is queued but not started. + RUNNING: Operation is currently in progress. + COMPLETED: Operation finished successfully. + FAILED: Operation encountered an error. + """ + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" ``` # Contributor Covenant Code of Conduct diff --git a/docs/api/conf.py b/docs/api/conf.py index a33caa10c..b69c62bbb 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -1,5 +1,6 @@ import logging import sys +from datetime import datetime from pathlib import Path # Configure logging @@ -13,7 +14,8 @@ sys.path.insert(0, str(project_root / "src")) # Project information project = "pipecat-ai" -copyright = "2024, Daily" +current_year = datetime.now().year +copyright = f"2024-{current_year}, Daily" if current_year > 2024 else "2024, Daily" author = "Daily" # General configuration @@ -26,16 +28,14 @@ extensions = [ # Napoleon settings napoleon_google_docstring = True -napoleon_numpy_docstring = False napoleon_include_init_with_doc = True # AutoDoc settings autodoc_default_options = { "members": True, "member-order": "bysource", - "special-members": "__init__", "undoc-members": True, - "exclude-members": "__weakref__", + "exclude-members": "__weakref__,model_config", "no-index": True, "show-inheritance": True, } @@ -145,12 +145,34 @@ autodoc_mock_imports = [ "transformers.AutoFeatureExtractor", # Also add specific classes that are imported "AutoFeatureExtractor", + # Sentry dependencies + "sentry_sdk", + # AWS Nova Sonic dependencies + "aws_sdk_bedrock_runtime", + "aws_sdk_bedrock_runtime.client", + "aws_sdk_bedrock_runtime.config", + "aws_sdk_bedrock_runtime.models", + "smithy_aws_core", + "smithy_aws_core.credentials_resolvers", + "smithy_aws_core.credentials_resolvers.static", + "smithy_aws_core.identity", + "smithy_core", + "smithy_core.aio", + "smithy_core.aio.eventstream", + # MCP dependencies (you may already have these) + "mcp", + "mcp.client", + "mcp.client.session_group", + "mcp.client.sse", + "mcp.client.stdio", + "mcp.ClientSession", + "mcp.StdioServerParameters", ] # HTML output settings html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] -autodoc_typehints = "description" +autodoc_typehints = "signature" # Show type hints in the signature only, not in the docstring html_show_sphinx = False @@ -249,6 +271,10 @@ def clean_title(title: str) -> str: "playht": "PlayHT", "xtts": "XTTS", "lmnt": "LMNT", + "stt": "STT", + "tts": "TTS", + "llm": "LLM", + "rtvi": "RTVI", } # Check if the entire title is a special case diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index e8613e082..319c97f78 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -61,7 +61,12 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si credentials=os.getenv("GOOGLE_TEST_CREDENTIALS"), ) - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY")) + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + model="gemini-2.5-flash", + # turn on thinking if you want it + # params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}),) + ) messages = [ { diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 9a7aa24b1..67701c53b 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -214,7 +214,12 @@ transport_params = { async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): logger.info(f"Starting bot") - llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"), model="gemini-2.0-flash-001") + llm = GoogleLLMService( + api_key=os.getenv("GOOGLE_API_KEY"), + model="gemini-2.5-flash", + # turn on thinking if you want it + # params=GoogleLLMService.InputParams(extra={"thinking_config": {"thinking_budget": 4096}}), + ) tts = GoogleTTSService( voice_id="en-US-Chirp3-HD-Charon", diff --git a/examples/foundational/39c-mcp-run-http.py b/examples/foundational/39c-mcp-run-http.py new file mode 100644 index 000000000..1c51894b1 --- /dev/null +++ b/examples/foundational/39c-mcp-run-http.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import argparse +import os + +from dotenv import load_dotenv +from loguru import logger +from mcp.client.session_group import StreamableHttpParameters + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia.tts import CartesiaTTSService +from pipecat.services.deepgram.stt import DeepgramSTTService +from pipecat.services.mcp_service import MCPClient +from pipecat.services.openai.llm import OpenAILLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.network.fastapi_websocket import FastAPIWebsocketParams +from pipecat.transports.services.daily import DailyParams + +load_dotenv(override=True) + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "twilio": lambda: FastAPIWebsocketParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_sigint: bool): + logger.info(f"Starting bot") + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o-mini") + + try: + # Github MCP docs: https://github.com/github/github-mcp-server + # Enable Github Copilot on your GitHub account. Free tier is ok. (https://github.com/settings/copilot) + # Generate a personal access token. It must be a Fine-grained token, classic tokens are not supported. (https://github.com/settings/personal-access-tokens) + # Set permissions you want to use (eg. "all repositories", "profile: read/write", etc) + mcp = MCPClient( + server_params=StreamableHttpParameters( + url="https://api.githubcopilot.com/mcp/", + headers={"Authorization": f"Bearer {os.getenv('GITHUB_PERSONAL_ACCESS_TOKEN')}"}, + ) + ) + except Exception as e: + logger.error(f"error setting up mcp") + logger.exception("error trace:") + + tools = await mcp.register_tools(llm) + + system = f""" + You are a helpful LLM in a WebRTC call. + Your goal is to answer questions about the user's GitHub repositories and account. + You have access to a number of tools provided by Github. Use any and all tools to help users. + Your output will be converted to audio so don't include special characters in your answers. + Don't overexplain what you are doing. + Just respond with short sentences when you are carrying out tool calls. + """ + + messages = [{"role": "system", "content": system}] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, + context_aggregator.user(), # User spoken responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses and tool context + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=handle_sigint) + + await runner.run(task) + + +if __name__ == "__main__": + from pipecat.examples.run import main + + main(run_example, transport_params=transport_params) diff --git a/examples/foundational/42-interruption-config.py b/examples/foundational/42-interruption-config.py index 9b57cc926..7f83b7023 100644 --- a/examples/foundational/42-interruption-config.py +++ b/examples/foundational/42-interruption-config.py @@ -10,8 +10,8 @@ import os from dotenv import load_dotenv from loguru import logger +from pipecat.audio.interruptions.min_words_interruption_strategy import MinWordsInterruptionStrategy from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import MinWordsInterruptionStrategy from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask diff --git a/examples/freeze-test/freeze_test_bot.py b/examples/freeze-test/freeze_test_bot.py index 5be79ad12..52ef5fc89 100644 --- a/examples/freeze-test/freeze_test_bot.py +++ b/examples/freeze-test/freeze_test_bot.py @@ -7,9 +7,11 @@ import argparse import asyncio import os +import random from contextlib import asynccontextmanager from typing import Any, Dict +import sentry_sdk import uvicorn from dotenv import load_dotenv from fastapi import FastAPI, Request, WebSocket @@ -44,6 +46,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import RTVIConfig, RTVIProcessor +from pipecat.processors.metrics.sentry import SentryMetrics from pipecat.serializers.protobuf import ProtobufFrameSerializer from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService @@ -125,6 +128,7 @@ class SimulateFreezeInput(FrameProcessor): self._send_frames_task = None async def _send_user_text(self, text: str): + self.reset_watchdog() # Emulation as if the user has spoken and the stt transcribed await self.push_frame(UserStartedSpeakingFrame()) await self.push_frame(StartInterruptionFrame()) @@ -149,14 +153,13 @@ class SimulateFreezeInput(FrameProcessor): logger.debug("SimulateFreezeInput _send_frames") await self._send_user_text("Tell me a brief history of Brazil!") await asyncio.sleep(3) - await self._send_user_text("") - break - # i += 1 - # if i >= 5: - # break + await self._send_user_text("and who has discovered it") + i += 1 + if i >= 20: + break # sleeping 1s before interrupting - # wait_time = random.uniform(1, 10) - # await asyncio.sleep(wait_time) + wait_time = random.uniform(1, 10) + await asyncio.sleep(wait_time) except Exception as e: logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})") @@ -176,6 +179,11 @@ async def run_example(websocket_client): ), ) + sentry_sdk.init( + dsn=os.getenv("SENTRY_DSN"), + traces_sample_rate=1.0, + ) + freeze = SimulateFreezeInput() stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) @@ -183,9 +191,13 @@ async def run_example(websocket_client): tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady + metrics=SentryMetrics(), ) - llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY")) + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + metrics=SentryMetrics(), + ) rtvi = RTVIProcessor(config=RTVIConfig(config=[])) @@ -247,6 +259,7 @@ async def run_example(websocket_client): }, ), ], + enable_watchdog_timers=True, ) @transport.event_handler("on_client_connected") diff --git a/pyproject.toml b/pyproject.toml index f7c73d49a..f402ddfd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ langchain = [ "langchain~=0.3.20", "langchain-community~=0.3.20", "langchain-ope livekit = [ "livekit~=0.22.0", "livekit-api~=0.8.2", "tenacity~=9.0.0" ] lmnt = [ "websockets~=13.1" ] local = [ "pyaudio~=0.2.14" ] -mcp = [ "mcp[cli]~=1.6.0" ] +mcp = [ "mcp[cli]~=1.9.4" ] mem0 = [ "mem0ai~=0.1.94" ] mlx-whisper = [ "mlx-whisper~=0.4.2" ] moondream = [ "einops~=0.8.0", "timm~=1.0.13", "transformers~=4.48.0" ] @@ -123,9 +123,9 @@ select = [ "D", # Docstring rules "I", # Import rules ] -# We ignore D107 because class docstrings already document __init__ parameters -# and our Sphinx configuration uses napoleon_include_init_with_doc=True -ignore = ["D107"] + +[tool.ruff.lint.per-file-ignores] +"**/__init__.py" = ["D104"] [tool.ruff.lint.pydocstyle] convention = "google" diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index 242dd6fc5..2e4d563c7 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -111,11 +111,16 @@ TESTS_26 = [ # ("26d-gemini-multimodal-live-text.py", PROMPT_SIMPLE_MATH, None), ] +TESTS_40 = [ + ("40-aws-nova-sonic.py", PROMPT_SIMPLE_MATH, None), +] + TESTS = [ *TESTS_07, *TESTS_14, *TESTS_19, *TESTS_26, + *TESTS_40, ] diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 4b2d934ab..3a602a3e2 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -453,8 +453,8 @@ class StartFrame(SystemFrame): allow_interruptions: bool = False enable_metrics: bool = False enable_usage_metrics: bool = False - report_only_initial_ttfb: bool = False interruption_strategies: List[BaseInterruptionStrategy] = field(default_factory=list) + report_only_initial_ttfb: bool = False @dataclass diff --git a/src/pipecat/metrics/metrics.py b/src/pipecat/metrics/metrics.py index 262254ffd..fbd5f9c8c 100644 --- a/src/pipecat/metrics/metrics.py +++ b/src/pipecat/metrics/metrics.py @@ -22,6 +22,7 @@ class LLMTokenUsage(BaseModel): total_tokens: int cache_read_input_tokens: Optional[int] = None cache_creation_input_tokens: Optional[int] = None + reasoning_tokens: Optional[int] = None class LLMUsageMetricsData(MetricsData): diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 3a08f44ed..300492cc5 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -21,6 +21,7 @@ from pipecat.frames.frames import ( from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue class ParallelPipelineSource(FrameProcessor): @@ -76,20 +77,36 @@ class ParallelPipeline(BasePipeline): if len(args) == 0: raise Exception(f"ParallelPipeline needs at least one argument") + self._args = args self._sources = [] self._sinks = [] + self._pipelines = [] + self._seen_ids = set() self._endframe_counter: Dict[int, int] = {} self._up_task = None self._down_task = None - self._up_queue = asyncio.Queue() - self._down_queue = asyncio.Queue() - self._pipelines = [] + # + # BasePipeline + # + + def processors_with_metrics(self) -> List[FrameProcessor]: + return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines)) + + # + # Frame processor + # + + async def setup(self, setup: FrameProcessorSetup): + await super().setup(setup) + + self._up_queue = WatchdogQueue(setup.task_manager) + self._down_queue = WatchdogQueue(setup.task_manager) logger.debug(f"Creating {self} pipelines") - for processors in args: + for processors in self._args: if not isinstance(processors, list): raise TypeError(f"ParallelPipeline argument {processors} is not a list") @@ -107,19 +124,6 @@ class ParallelPipeline(BasePipeline): logger.debug(f"Finished creating {self} pipelines") - # - # BasePipeline - # - - def processors_with_metrics(self) -> List[FrameProcessor]: - return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines)) - - # - # Frame processor - # - - async def setup(self, setup: FrameProcessorSetup): - await super().setup(setup) await asyncio.gather(*[s.setup(setup) for s in self._sources]) await asyncio.gather(*[p.setup(setup) for p in self._pipelines]) await asyncio.gather(*[s.setup(setup) for s in self._sinks]) @@ -134,7 +138,7 @@ class ParallelPipeline(BasePipeline): await super().process_frame(frame, direction) if isinstance(frame, StartFrame): - await self._start() + await self._start(frame) elif isinstance(frame, EndFrame): self._endframe_counter[frame.id] = len(self._pipelines) elif isinstance(frame, CancelFrame): @@ -154,7 +158,7 @@ class ParallelPipeline(BasePipeline): elif isinstance(frame, EndFrame): await self._stop() - async def _start(self): + async def _start(self, frame: StartFrame): await self._create_tasks() async def _stop(self): @@ -202,18 +206,14 @@ class ParallelPipeline(BasePipeline): async def _process_up_queue(self): while True: frame = await self._up_queue.get() - self.start_watchdog() await self._parallel_push_frame(frame, FrameDirection.UPSTREAM) self._up_queue.task_done() - self.reset_watchdog() async def _process_down_queue(self): running = True while running: frame = await self._down_queue.get() - self.start_watchdog() - endframe_counter = self._endframe_counter.get(frame.id, 0) # If we have a counter, decrement it. @@ -228,5 +228,3 @@ class ParallelPipeline(BasePipeline): running = not (endframe_counter == 0 and isinstance(frame, EndFrame)) self._down_queue.task_done() - - self.reset_watchdog() diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py index 4cf9f5033..006290710 100644 --- a/src/pipecat/pipeline/sync_parallel_pipeline.py +++ b/src/pipecat/pipeline/sync_parallel_pipeline.py @@ -15,6 +15,7 @@ from pipecat.frames.frames import ControlFrame, EndFrame, Frame, SystemFrame from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue @dataclass @@ -61,15 +62,30 @@ class SyncParallelPipeline(BasePipeline): if len(args) == 0: raise Exception(f"SyncParallelPipeline needs at least one argument") + self._args = args self._sinks = [] self._sources = [] self._pipelines = [] - self._up_queue = asyncio.Queue() - self._down_queue = asyncio.Queue() + # + # BasePipeline + # + + def processors_with_metrics(self) -> List[FrameProcessor]: + return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines)) + + # + # Frame processor + # + + async def setup(self, setup: FrameProcessorSetup): + await super().setup(setup) + + self._up_queue = WatchdogQueue(setup.task_manager) + self._down_queue = WatchdogQueue(setup.task_manager) logger.debug(f"Creating {self} pipelines") - for processors in args: + for processors in self._args: if not isinstance(processors, list): raise TypeError(f"SyncParallelPipeline argument {processors} is not a list") @@ -92,19 +108,6 @@ class SyncParallelPipeline(BasePipeline): logger.debug(f"Finished creating {self} pipelines") - # - # BasePipeline - # - - def processors_with_metrics(self) -> List[FrameProcessor]: - return list(chain.from_iterable(p.processors_with_metrics() for p in self._pipelines)) - - # - # Frame processor - # - - async def setup(self, setup: FrameProcessorSetup): - await super().setup(setup) await asyncio.gather(*[s["processor"].setup(setup) for s in self._sources]) await asyncio.gather(*[p.setup(setup) for p in self._pipelines]) await asyncio.gather(*[s["processor"].setup(setup) for s in self._sinks]) diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 88a47189b..0cf294b05 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -38,7 +38,13 @@ from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.base_task import BasePipelineTask, PipelineTaskParams from pipecat.pipeline.task_observer import TaskObserver from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, FrameProcessorSetup -from pipecat.utils.asyncio import WATCHDOG_TIMEOUT, BaseTaskManager, TaskManager, TaskManagerParams +from pipecat.utils.asyncio.task_manager import ( + WATCHDOG_TIMEOUT, + BaseTaskManager, + TaskManager, + TaskManagerParams, +) +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue from pipecat.utils.tracing.setup import is_tracing_available from pipecat.utils.tracing.turn_trace_observer import TurnTraceObserver @@ -188,6 +194,7 @@ class PipelineTask(BasePipelineTask): enable_tracing: Whether to enable tracing. enable_turn_tracking: Whether to enable turn tracking. enable_watchdog_logging: Whether to print task processing times. + enable_watchdog_timers: Whether to enable task watchdog timers. idle_timeout_frames: A tuple with the frames that should trigger an idle timeout if not received withing `idle_timeout_seconds`. idle_timeout_secs: Timeout (in seconds) to consider pipeline idle or @@ -211,6 +218,7 @@ class PipelineTask(BasePipelineTask): enable_tracing: bool = False, enable_turn_tracking: bool = True, enable_watchdog_logging: bool = False, + enable_watchdog_timers: bool = False, idle_timeout_frames: Tuple[Type[Frame], ...] = ( BotSpeakingFrame, LLMFullResponseEndFrame, @@ -231,6 +239,7 @@ class PipelineTask(BasePipelineTask): self._enable_tracing = enable_tracing and is_tracing_available() self._enable_turn_tracking = enable_turn_tracking self._enable_watchdog_logging = enable_watchdog_logging + self._enable_watchdog_timers = enable_watchdog_timers self._idle_timeout_frames = idle_timeout_frames self._idle_timeout_secs = idle_timeout_secs self._watchdog_timeout_secs = watchdog_timeout_secs @@ -260,19 +269,29 @@ class PipelineTask(BasePipelineTask): self._finished = False self._cancelled = False + # This task maneger will handle all the asyncio tasks created by this + # PipelineTask and its frame processors. + self._task_manager = task_manager or TaskManager() + # This queue receives frames coming from the pipeline upstream. - self._up_queue = asyncio.Queue() + self._up_queue = WatchdogQueue(self._task_manager) + self._process_up_task: Optional[asyncio.Task] = None # This queue receives frames coming from the pipeline downstream. - self._down_queue = asyncio.Queue() + self._down_queue = WatchdogQueue(self._task_manager) + self._process_down_task: Optional[asyncio.Task] = None # This queue is the queue used to push frames to the pipeline. - self._push_queue = asyncio.Queue() + self._push_queue = WatchdogQueue(self._task_manager) + self._process_push_task: Optional[asyncio.Task] = None # This is the heartbeat queue. When a heartbeat frame is received in the # down queue we add it to the heartbeat queue for processing. - self._heartbeat_queue = asyncio.Queue() + self._heartbeat_queue = WatchdogQueue(self._task_manager) + self._heartbeat_push_task: Optional[asyncio.Task] = None + self._heartbeat_monitor_task: Optional[asyncio.Task] = None # This is the idle queue. When frames are received downstream they are # put in the queue. If no frame is received the pipeline is considered # idle. - self._idle_queue = asyncio.Queue() + self._idle_queue = WatchdogQueue(self._task_manager) + self._idle_monitor_task: Optional[asyncio.Task] = None # This event is used to indicate a finalize frame (e.g. EndFrame, # StopFrame) has been received in the down queue. self._pipeline_end_event = asyncio.Event() @@ -289,10 +308,6 @@ class PipelineTask(BasePipelineTask): self._sink = PipelineTaskSink(self._down_queue) pipeline.link(self._sink) - # This task maneger will handle all the asyncio tasks created by this - # PipelineTask and its frame processors. - self._task_manager = task_manager or TaskManager() - # The task observer acts as a proxy to the provided observers. This way, # we only need to pass a single observer (using the StartFrame) which # then just acts as a proxy. @@ -433,7 +448,9 @@ class PipelineTask(BasePipelineTask): # we want to cancel right away. await self._source.push_frame(CancelFrame()) # Only cancel the push task. Everything else will be cancelled in run(). - await self._task_manager.cancel_task(self._process_push_task) + if self._process_push_task: + await self._task_manager.cancel_task(self._process_push_task) + self._process_push_task = None async def _create_tasks(self): self._process_up_task = self._task_manager.create_task( @@ -451,7 +468,7 @@ class PipelineTask(BasePipelineTask): return self._process_push_task def _maybe_start_heartbeat_tasks(self): - if self._params.enable_heartbeats: + if self._params.enable_heartbeats and self._heartbeat_push_task is None: self._heartbeat_push_task = self._task_manager.create_task( self._heartbeat_push_handler(), f"{self}::_heartbeat_push_handler" ) @@ -468,20 +485,33 @@ class PipelineTask(BasePipelineTask): async def _cancel_tasks(self): await self._observer.stop() - await self._task_manager.cancel_task(self._process_up_task) - await self._task_manager.cancel_task(self._process_down_task) + if self._process_up_task: + await self._task_manager.cancel_task(self._process_up_task) + self._process_up_task = None + + if self._process_down_task: + await self._task_manager.cancel_task(self._process_down_task) + self._process_down_task = None await self._maybe_cancel_heartbeat_tasks() await self._maybe_cancel_idle_task() async def _maybe_cancel_heartbeat_tasks(self): - if self._params.enable_heartbeats: + if not self._params.enable_heartbeats: + return + + if self._heartbeat_push_task: await self._task_manager.cancel_task(self._heartbeat_push_task) + self._heartbeat_push_task = None + + if self._heartbeat_monitor_task: await self._task_manager.cancel_task(self._heartbeat_monitor_task) + self._heartbeat_monitor_task = None async def _maybe_cancel_idle_task(self): - if self._idle_timeout_secs: + if self._idle_timeout_secs and self._idle_monitor_task: await self._task_manager.cancel_task(self._idle_monitor_task) + self._idle_monitor_task = None def _initial_metrics_frame(self) -> MetricsFrame: processors = self._pipeline.processors_with_metrics() @@ -499,6 +529,7 @@ class PipelineTask(BasePipelineTask): mgr_params = TaskManagerParams( loop=params.loop, enable_watchdog_logging=self._enable_watchdog_logging, + enable_watchdog_timers=self._enable_watchdog_timers, watchdog_timeout=self._watchdog_timeout_secs, ) self._task_manager.setup(mgr_params) @@ -507,6 +538,7 @@ class PipelineTask(BasePipelineTask): clock=self._clock, task_manager=self._task_manager, observer=self._observer, + watchdog_timers_enabled=self._enable_watchdog_timers, ) await self._source.setup(setup) await self._pipeline.setup(setup) @@ -526,8 +558,6 @@ class PipelineTask(BasePipelineTask): await self._pipeline.cleanup() await self._sink.cleanup() - await self._task_manager.cleanup() - async def _process_push_queue(self): """This is the task that runs the pipeline for the first time by sending a StartFrame and by pushing any other frames queued by the user. It runs @@ -536,7 +566,6 @@ class PipelineTask(BasePipelineTask): """ self._clock.start() - self._maybe_start_heartbeat_tasks() self._maybe_start_idle_task() start_frame = StartFrame( @@ -618,6 +647,10 @@ class PipelineTask(BasePipelineTask): if isinstance(frame, StartFrame): await self._call_event_handler("on_pipeline_started", frame) + + # Start heartbeat tasks now that StartFrame has been processed + # by all processors in the pipeline + self._maybe_start_heartbeat_tasks() elif isinstance(frame, EndFrame): await self._call_event_handler("on_pipeline_ended", frame) self._pipeline_end_event.set() diff --git a/src/pipecat/pipeline/task_observer.py b/src/pipecat/pipeline/task_observer.py index 950ddc43b..40f4c953c 100644 --- a/src/pipecat/pipeline/task_observer.py +++ b/src/pipecat/pipeline/task_observer.py @@ -11,7 +11,8 @@ from typing import Dict, List, Optional from attr import dataclass from pipecat.observers.base_observer import BaseObserver, FramePushed -from pipecat.utils.asyncio import BaseTaskManager +from pipecat.utils.asyncio.task_manager import BaseTaskManager +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue @dataclass @@ -82,6 +83,9 @@ class TaskObserver(BaseObserver): async def stop(self): """Stops all proxy observer tasks.""" + if not self._proxies: + return + for proxy in self._proxies.values(): await self._task_manager.cancel_task(proxy.task) @@ -93,7 +97,7 @@ class TaskObserver(BaseObserver): return self._proxies is not None def _create_proxy(self, observer: BaseObserver) -> Proxy: - queue = asyncio.Queue() + queue = WatchdogQueue(self._task_manager) task = self._task_manager.create_task( self._proxy_task_handler(queue, observer), f"TaskObserver::{observer}::_proxy_task_handler", diff --git a/src/pipecat/processors/aggregators/dtmf_aggregator.py b/src/pipecat/processors/aggregators/dtmf_aggregator.py index cc6218a1f..006a7c181 100644 --- a/src/pipecat/processors/aggregators/dtmf_aggregator.py +++ b/src/pipecat/processors/aggregators/dtmf_aggregator.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""DTMF aggregation processor for converting keypad input to transcription. + +This module provides a frame processor that aggregates DTMF (Dual-Tone Multi-Frequency) +keypad inputs into meaningful sequences and converts them to transcription frames +for downstream processing by LLM context aggregators. +""" + import asyncio from typing import Optional @@ -31,11 +38,6 @@ class DTMFAggregator(FrameProcessor): - EndFrame or CancelFrame is received Emits TranscriptionFrame for compatibility with existing LLM context aggregators. - - Args: - timeout: Idle timeout in seconds before flushing - termination_digit: Digit that triggers immediate flush - prefix: Prefix added to DTMF sequence in transcription """ def __init__( @@ -45,6 +47,14 @@ class DTMFAggregator(FrameProcessor): prefix: str = "DTMF: ", **kwargs, ): + """Initialize the DTMF aggregator. + + Args: + timeout: Idle timeout in seconds before flushing + termination_digit: Digit that triggers immediate flush + prefix: Prefix added to DTMF sequence in transcription + **kwargs: Additional arguments passed to FrameProcessor + """ super().__init__(**kwargs) self._aggregation = "" self._idle_timeout = timeout @@ -55,6 +65,12 @@ class DTMFAggregator(FrameProcessor): self._aggregation_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection) -> None: + """Process incoming frames and handle DTMF aggregation. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -119,6 +135,7 @@ class DTMFAggregator(FrameProcessor): await asyncio.wait_for(self._digit_event.wait(), timeout=self._idle_timeout) self._digit_event.clear() except asyncio.TimeoutError: + self.reset_watchdog() if self._aggregation: await self._flush_aggregation() diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index 3c8aac26e..33f80247d 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Gated frame aggregator for conditional frame accumulation. + +This module provides a gated aggregator that accumulates frames based on +custom gate open/close functions, allowing for conditional frame buffering +and release in frame processing pipelines. +""" + from typing import List, Tuple from loguru import logger @@ -14,8 +21,11 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class GatedAggregator(FrameProcessor): """Accumulate frames, with custom functions to start and stop accumulation. + Yields gate-opening frame before any accumulated frames, then ensuing frames - until and not including the gate-closed frame. + until and not including the gate-closed frame. The aggregator maintains an + internal gate state that controls whether frames are passed through immediately + or accumulated for later release. Doctest: FIXME to work with asyncio >>> from pipecat.frames.frames import ImageRawFrame @@ -48,6 +58,14 @@ class GatedAggregator(FrameProcessor): start_open, direction: FrameDirection = FrameDirection.DOWNSTREAM, ): + """Initialize the gated aggregator. + + Args: + gate_open_fn: Function that returns True when a frame should open the gate. + gate_close_fn: Function that returns True when a frame should close the gate. + start_open: Whether the gate should start in the open state. + direction: The frame direction this aggregator operates on. + """ super().__init__() self._gate_open_fn = gate_open_fn self._gate_close_fn = gate_close_fn @@ -56,6 +74,12 @@ class GatedAggregator(FrameProcessor): self._accumulator: List[Tuple[Frame, FrameDirection]] = [] async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames with gated accumulation logic. + + Args: + frame: The frame to process. + direction: The direction of the frame flow. + """ await super().process_frame(frame, direction) # We must not block system frames. diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index 9973e3d02..56423403d 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Gated OpenAI LLM context aggregator for controlled message flow.""" + from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -11,12 +13,21 @@ from pipecat.sync.base_notifier import BaseNotifier class GatedOpenAILLMContextAggregator(FrameProcessor): - """This aggregator keeps the last received OpenAI LLM context frame and it - doesn't let it through until the notifier is notified. + """Aggregator that gates OpenAI LLM context frames until notified. + This aggregator captures OpenAI LLM context frames and holds them until + a notifier signals that they can be released. This is useful for controlling + the flow of context frames based on external conditions or timing. """ def __init__(self, *, notifier: BaseNotifier, start_open: bool = False, **kwargs): + """Initialize the gated context aggregator. + + Args: + notifier: The notifier that controls when frames are released. + start_open: If True, the first context frame passes through immediately. + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) self._notifier = notifier self._start_open = start_open @@ -24,6 +35,12 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): self._gate_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames, gating OpenAI LLM context frames. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -42,15 +59,18 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): await self.push_frame(frame, direction) async def _start(self): + """Start the gate task handler.""" if not self._gate_task: self._gate_task = self.create_task(self._gate_task_handler()) async def _stop(self): + """Stop the gate task handler.""" if self._gate_task: await self.cancel_task(self._gate_task) self._gate_task = None async def _gate_task_handler(self): + """Handle the gating logic by waiting for notifications and releasing frames.""" while True: await self._notifier.wait() if self._last_context_frame: diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 1984e3cf8..6d27d1ddf 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""LLM response aggregators for handling conversation context and message aggregation. + +This module provides aggregators that process and accumulate LLM responses, user inputs, +and conversation context. These aggregators handle the flow between speech-to-text, +LLM processing, and text-to-speech components in conversational AI pipelines. +""" + import asyncio from abc import abstractmethod from dataclasses import dataclass @@ -54,30 +61,55 @@ from pipecat.utils.time import time_now_iso8601 @dataclass class LLMUserAggregatorParams: + """Parameters for configuring LLM user aggregation behavior. + + Parameters: + aggregation_timeout: Maximum time in seconds to wait for additional + transcription content before pushing aggregated result. This + timeout is used only when the transcription is slow to arrive. + """ + aggregation_timeout: float = 0.5 @dataclass class LLMAssistantAggregatorParams: + """Parameters for configuring LLM assistant aggregation behavior. + + Parameters: + expect_stripped_words: Whether to expect and handle stripped words + in text frames by adding spaces between tokens. + """ + expect_stripped_words: bool = True class LLMFullResponseAggregator(FrameProcessor): - """This is an LLM aggregator that aggregates a full LLM completion. It - aggregates LLM text frames (tokens) received between - `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. Every full - completion is returned via the "on_completion" event handler: + """Aggregates complete LLM responses between start and end frames. - @aggregator.event_handler("on_completion") - async def on_completion( - aggregator: LLMFullResponseAggregator, - completion: str, - completed: bool, - ) + This aggregator collects LLM text frames (tokens) received between + `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame` and provides + the complete response via an event handler. + The aggregator provides an "on_completion" event that fires when a full + completion is available: + + @aggregator.event_handler("on_completion") + async def on_completion( + aggregator: LLMFullResponseAggregator, + completion: str, + completed: bool, + ): + # Handle the completion + pass """ def __init__(self, **kwargs): + """Initialize the LLM full response aggregator. + + Args: + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) self._aggregation = "" @@ -86,6 +118,12 @@ class LLMFullResponseAggregator(FrameProcessor): self._register_event_handler("on_completion") async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and aggregate LLM text content. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): @@ -116,83 +154,123 @@ class LLMFullResponseAggregator(FrameProcessor): class BaseLLMResponseAggregator(FrameProcessor): - """This is the base class for all LLM response aggregators. These - aggregators process incoming frames and aggregate content until they are - ready to push the aggregation. In the case of a user, an aggregation might - be a full transcription received from the STT service. + """Base class for all LLM response aggregators. - The LLM response aggregators also keep a store (e.g. a message list or an - LLM context) of the current conversation, that is, it stores the messages - said by the user or by the bot. + These aggregators process incoming frames and aggregate content until they are + ready to push the aggregation downstream. They maintain conversation state + and handle message flow between different components in the pipeline. + The aggregators keep a store (e.g. message list or LLM context) of the current + conversation, storing messages from both users and the bot. """ def __init__(self, **kwargs): + """Initialize the base LLM response aggregator. + + Args: + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) @property @abstractmethod def messages(self) -> List[dict]: - """Returns the messages from the current conversation.""" + """Get the messages from the current conversation. + + Returns: + List of message dictionaries representing the conversation history. + """ pass @property @abstractmethod def role(self) -> str: - """Returns the role (e.g. user, assistant...) for this aggregator.""" + """Get the role for this aggregator. + + Returns: + The role string (e.g. "user", "assistant") for this aggregator. + """ pass @abstractmethod def add_messages(self, messages): - """Add the given messages to the conversation.""" + """Add the given messages to the conversation. + + Args: + messages: Messages to append to the conversation history. + """ pass @abstractmethod def set_messages(self, messages): - """Reset the conversation with the given messages.""" + """Reset the conversation with the given messages. + + Args: + messages: Messages to replace the current conversation history. + """ pass @abstractmethod def set_tools(self, tools): - """Set LLM tools to be used in the current conversation.""" + """Set LLM tools to be used in the current conversation. + + Args: + tools: List of tool definitions for the LLM to use. + """ pass @abstractmethod def set_tool_choice(self, tool_choice): - """Set the tool choice. This should modify the LLM context.""" + """Set the tool choice for the LLM. + + Args: + tool_choice: Tool choice configuration for the LLM context. + """ pass @abstractmethod async def reset(self): - """Reset the internals of this aggregator. This should not modify the - internal messages. + """Reset the internal state of this aggregator. + + This should clear aggregation state but not modify the conversation messages. """ pass @abstractmethod async def handle_aggregation(self, aggregation: str): - """Adds the given aggregation to the aggregator. The aggregator can use - a simple list of message or a context. It doesn't not push any frames. + """Add the given aggregation to the conversation store. + Args: + aggregation: The aggregated text content to add to the conversation. """ pass @abstractmethod async def push_aggregation(self): - """Pushes the current aggregation. For example, iN the case of context - aggregation this might push a new context frame. + """Push the current aggregation downstream. + The specific frame type pushed depends on the aggregator implementation + (e.g. context frame, messages frame). """ pass class LLMContextResponseAggregator(BaseLLMResponseAggregator): - """This is a base LLM aggregator that uses an LLM context to store the - conversation. It pushes `OpenAILLMContextFrame` as an aggregation frame. + """Base LLM aggregator that uses an OpenAI LLM context for conversation storage. + This aggregator maintains conversation state using an OpenAILLMContext and + pushes OpenAILLMContextFrame objects as aggregation frames. It provides + common functionality for context-based conversation management. """ def __init__(self, *, context: OpenAILLMContext, role: str, **kwargs): + """Initialize the context response aggregator. + + Args: + context: The OpenAI LLM context to use for conversation storage. + role: The role this aggregator represents (e.g. "user", "assistant"). + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._context = context self._role = role @@ -201,46 +279,98 @@ class LLMContextResponseAggregator(BaseLLMResponseAggregator): @property def messages(self) -> List[dict]: + """Get messages from the LLM context. + + Returns: + List of message dictionaries from the context. + """ return self._context.get_messages() @property def role(self) -> str: + """Get the role for this aggregator. + + Returns: + The role string for this aggregator. + """ return self._role @property def context(self): + """Get the OpenAI LLM context. + + Returns: + The OpenAILLMContext instance used by this aggregator. + """ return self._context def get_context_frame(self) -> OpenAILLMContextFrame: + """Create a context frame with the current context. + + Returns: + OpenAILLMContextFrame containing the current context. + """ return OpenAILLMContextFrame(context=self._context) async def push_context_frame(self, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a context frame in the specified direction. + + Args: + direction: The direction to push the frame (upstream or downstream). + """ frame = self.get_context_frame() await self.push_frame(frame, direction) def add_messages(self, messages): + """Add messages to the context. + + Args: + messages: Messages to add to the conversation context. + """ self._context.add_messages(messages) def set_messages(self, messages): + """Set the context messages. + + Args: + messages: Messages to replace the current context messages. + """ self._context.set_messages(messages) def set_tools(self, tools: List): + """Set tools in the context. + + Args: + tools: List of tool definitions to set in the context. + """ self._context.set_tools(tools) def set_tool_choice(self, tool_choice: Literal["none", "auto", "required"] | dict): + """Set tool choice in the context. + + Args: + tool_choice: Tool choice configuration for the context. + """ self._context.set_tool_choice(tool_choice) async def reset(self): + """Reset the aggregation state.""" self._aggregation = "" class LLMUserContextAggregator(LLMContextResponseAggregator): - """This is a user LLM aggregator that uses an LLM context to store the - conversation. It aggregates transcriptions from the STT service and it has - logic to handle multiple scenarios where transcriptions are received between - VAD events (`UserStartedSpeakingFrame` and `UserStoppedSpeakingFrame`) or - even outside or no VAD events at all. + """User LLM aggregator that processes speech-to-text transcriptions. + This aggregator handles the complex logic of aggregating user speech transcriptions + from STT services. It manages multiple scenarios including: + - Transcriptions received between VAD events + - Transcriptions received outside VAD events + - Interim vs final transcriptions + - User interruptions during bot speech + - Emulated VAD for whispered or short utterances + + The aggregator uses timeouts to handle cases where transcriptions arrive + after VAD events or when no VAD is available. """ def __init__( @@ -250,6 +380,13 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): params: Optional[LLMUserAggregatorParams] = None, **kwargs, ): + """Initialize the user context aggregator. + + Args: + context: The OpenAI LLM context for conversation storage. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments. Supports deprecated 'aggregation_timeout'. + """ super().__init__(context=context, role="user", **kwargs) self._params = params or LLMUserAggregatorParams() if "aggregation_timeout" in kwargs: @@ -275,6 +412,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): self._aggregation_task = None async def reset(self): + """Reset the aggregation state and interruption strategies.""" await super().reset() self._was_bot_speaking = False self._seen_interim_results = False @@ -282,9 +420,20 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): [await s.reset() for s in self._interruption_strategies] async def handle_aggregation(self, aggregation: str): + """Add the aggregated user text to the context. + + Args: + aggregation: The aggregated user text to add as a user message. + """ self._context.add_message({"role": self.role, "content": aggregation}) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for user speech aggregation and context management. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -339,7 +488,7 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): await self.push_frame(frame) async def push_aggregation(self): - """Pushes the current aggregation based on interruption strategies and conditions.""" + """Push the current aggregation based on interruption strategies and conditions.""" if len(self._aggregation) > 0: if self.interruption_strategies and self._bot_speaking: should_interrupt = await self._should_interrupt_based_on_strategies() @@ -373,7 +522,11 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): # await self.push_frame(OpenAILLMContextFrame(self._context)) async def _should_interrupt_based_on_strategies(self) -> bool: - """Check if interruption should occur based on configured strategies.""" + """Check if interruption should occur based on configured strategies. + + Returns: + True if any interruption strategy indicates interruption should occur. + """ async def should_interrupt(strategy: BaseInterruptionStrategy): await strategy.append_text(self._aggregation) @@ -470,12 +623,14 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): ) self._emulating_vad = False finally: + self.reset_watchdog() self._aggregation_event.clear() async def _maybe_emulate_user_speaking(self): - """Emulate user speaking if we got a transcription but it was not - detected by VAD. Only do that if the bot is not speaking. + """Maybe emulate user speaking based on transcription. + Emulate user speaking if we got a transcription but it was not + detected by VAD. Only do that if the bot is not speaking. """ # Check if we received a transcription but VAD was not able to detect # voice (e.g. when you whisper a short utterance). In that case, we need @@ -496,10 +651,17 @@ class LLMUserContextAggregator(LLMContextResponseAggregator): class LLMAssistantContextAggregator(LLMContextResponseAggregator): - """This is an assistant LLM aggregator that uses an LLM context to store the - conversation. It aggregates text frames received between - `LLMFullResponseStartFrame` and `LLMFullResponseEndFrame`. + """Assistant LLM aggregator that processes bot responses and function calls. + This aggregator handles the complex logic of processing assistant responses including: + - Text frame aggregation between response start/end markers + - Function call lifecycle management + - Context updates with timestamps + - Tool execution and result handling + - Interruption handling during responses + + The aggregator manages function calls in progress and coordinates between + text generation and tool execution phases of LLM responses. """ def __init__( @@ -509,6 +671,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): params: Optional[LLMAssistantAggregatorParams] = None, **kwargs, ): + """Initialize the assistant context aggregator. + + Args: + context: The OpenAI LLM context for conversation storage. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments. Supports deprecated 'expect_stripped_words'. + """ super().__init__(context=context, role="assistant", **kwargs) self._params = params or LLMAssistantAggregatorParams() @@ -533,26 +702,57 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): """Check if there are any function calls currently in progress. Returns: - bool: True if function calls are in progress, False otherwise + True if function calls are in progress, False otherwise. """ return bool(self._function_calls_in_progress) async def handle_aggregation(self, aggregation: str): + """Add the aggregated assistant text to the context. + + Args: + aggregation: The aggregated assistant text to add as an assistant message. + """ self._context.add_message({"role": "assistant", "content": aggregation}) async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + """Handle a function call that is in progress. + + Args: + frame: The function call in progress frame to handle. + """ pass async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle the result of a completed function call. + + Args: + frame: The function call result frame to handle. + """ pass async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + """Handle cancellation of a function call. + + Args: + frame: The function call cancel frame to handle. + """ pass async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle a user image frame associated with a function call. + + Args: + frame: The user image frame to handle. + """ pass async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for assistant response aggregation and function call management. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): @@ -589,6 +789,7 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): await self.push_frame(frame, direction) async def push_aggregation(self): + """Push the current assistant aggregation with timestamp.""" if not self._aggregation: return @@ -718,6 +919,13 @@ class LLMAssistantContextAggregator(LLMContextResponseAggregator): class LLMUserResponseAggregator(LLMUserContextAggregator): + """User response aggregator that outputs LLMMessagesFrame instead of context frames. + + This aggregator extends LLMUserContextAggregator but pushes LLMMessagesFrame + objects downstream instead of OpenAILLMContextFrame objects. This is useful + when you need message-based output rather than context-based output. + """ + def __init__( self, messages: Optional[List[dict]] = None, @@ -725,9 +933,17 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): params: Optional[LLMUserAggregatorParams] = None, **kwargs, ): + """Initialize the user response aggregator. + + Args: + messages: Initial messages for the conversation context. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs) async def push_aggregation(self): + """Push the aggregated user input as an LLMMessagesFrame.""" if len(self._aggregation) > 0: await self.handle_aggregation(self._aggregation) @@ -740,6 +956,13 @@ class LLMUserResponseAggregator(LLMUserContextAggregator): class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): + """Assistant response aggregator that outputs LLMMessagesFrame instead of context frames. + + This aggregator extends LLMAssistantContextAggregator but pushes LLMMessagesFrame + objects downstream instead of OpenAILLMContextFrame objects. This is useful + when you need message-based output rather than context-based output. + """ + def __init__( self, messages: Optional[List[dict]] = None, @@ -747,9 +970,17 @@ class LLMAssistantResponseAggregator(LLMAssistantContextAggregator): params: Optional[LLMAssistantAggregatorParams] = None, **kwargs, ): + """Initialize the assistant response aggregator. + + Args: + messages: Initial messages for the conversation context. + params: Configuration parameters for aggregation behavior. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(context=OpenAILLMContext(messages), params=params, **kwargs) async def push_aggregation(self): + """Push the aggregated assistant response as an LLMMessagesFrame.""" if len(self._aggregation) > 0: await self.handle_aggregation(self._aggregation) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 806741c4c..a8520546a 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI LLM context management for Pipecat. + +This module provides classes for managing OpenAI-specific conversation contexts, +including message handling, tool management, and image/audio processing capabilities. +""" + import base64 import copy import io @@ -29,7 +35,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class CustomEncoder(json.JSONEncoder): + """Custom JSON encoder for handling special data types in logging. + + Provides specialized encoding for io.BytesIO objects to display + readable representations in log output instead of raw binary data. + """ + def default(self, obj): + """Encode special objects for JSON serialization. + + Args: + obj: The object to encode. + + Returns: + Encoded representation of the object. + """ if isinstance(obj, io.BytesIO): # Convert the first 8 bytes to an ASCII hex string return f"{obj.getbuffer()[0:8].hex()}..." @@ -37,25 +57,57 @@ class CustomEncoder(json.JSONEncoder): class OpenAILLMContext: + """Manages conversation context for OpenAI LLM interactions. + + Handles message history, tool definitions, tool choices, and multimedia content + for OpenAI API conversations. Provides methods for message manipulation, + content formatting, and integration with various LLM adapters. + """ + def __init__( self, messages: Optional[List[ChatCompletionMessageParam]] = None, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = NOT_GIVEN, ): + """Initialize the OpenAI LLM context. + + Args: + messages: Initial list of conversation messages. + tools: Available tools for the LLM to use. + tool_choice: Tool selection strategy for the LLM. + """ self._messages: List[ChatCompletionMessageParam] = messages if messages else [] self._tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven = tool_choice self._tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = tools self._llm_adapter: Optional[BaseLLMAdapter] = None def get_llm_adapter(self) -> Optional[BaseLLMAdapter]: + """Get the current LLM adapter. + + Returns: + The currently set LLM adapter, or None if not set. + """ return self._llm_adapter def set_llm_adapter(self, llm_adapter: BaseLLMAdapter): + """Set the LLM adapter for context processing. + + Args: + llm_adapter: The LLM adapter to use for tool conversion. + """ self._llm_adapter = llm_adapter @staticmethod def from_messages(messages: List[dict]) -> "OpenAILLMContext": + """Create a context from a list of message dictionaries. + + Args: + messages: List of message dictionaries to convert to context. + + Returns: + New OpenAILLMContext instance with the provided messages. + """ context = OpenAILLMContext() for message in messages: @@ -66,34 +118,81 @@ class OpenAILLMContext: @property def messages(self) -> List[ChatCompletionMessageParam]: + """Get the current messages list. + + Returns: + List of conversation messages. + """ return self._messages @property def tools(self) -> List[ChatCompletionToolParam] | NotGiven | List[Any]: + """Get the tools list, converting through adapter if available. + + Returns: + Tools list, potentially converted by the LLM adapter. + """ if self._llm_adapter: return self._llm_adapter.from_standard_tools(self._tools) return self._tools @property def tool_choice(self) -> ChatCompletionToolChoiceOptionParam | NotGiven: + """Get the current tool choice setting. + + Returns: + The tool choice configuration. + """ return self._tool_choice def add_message(self, message: ChatCompletionMessageParam): + """Add a single message to the context. + + Args: + message: The message to add to the conversation history. + """ self._messages.append(message) def add_messages(self, messages: List[ChatCompletionMessageParam]): + """Add multiple messages to the context. + + Args: + messages: List of messages to add to the conversation history. + """ self._messages.extend(messages) def set_messages(self, messages: List[ChatCompletionMessageParam]): + """Replace all messages in the context. + + Args: + messages: New list of messages to replace the current history. + """ self._messages[:] = messages def get_messages(self) -> List[ChatCompletionMessageParam]: + """Get a copy of the current messages list. + + Returns: + List of all messages in the conversation history. + """ return self._messages def get_messages_json(self) -> str: + """Get messages as a formatted JSON string. + + Returns: + JSON string representation of all messages with custom encoding. + """ return json.dumps(self._messages, cls=CustomEncoder, ensure_ascii=False, indent=2) def get_messages_for_logging(self) -> str: + """Get sanitized messages suitable for logging. + + Removes or truncates sensitive data like image content for safe logging. + + Returns: + JSON string with sanitized message content for logging. + """ msgs = [] for message in self.messages: msg = copy.deepcopy(message) @@ -118,10 +217,10 @@ class OpenAILLMContext: Since OpenAI is our standard format, this is a passthrough function. Args: - message (dict): Message in OpenAI format + message: Message in OpenAI format. Returns: - dict: Same message, unchanged + Same message, unchanged. """ return message @@ -133,20 +232,28 @@ class OpenAILLMContext: other LLM services that may need to return multiple messages. Args: - obj (dict): Message in OpenAI format with either: - - Simple content: {"role": "user", "content": "Hello"} - - List content: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + obj: Message in OpenAI format with either simple string content + or structured list content. Returns: - list: List containing the original messages, preserving whether - the content was in simple string or structured list format + List containing the original messages, preserving the content format. """ return [obj] def get_messages_for_initializing_history(self): + """Get messages for initializing conversation history. + + Returns: + List of messages suitable for history initialization. + """ return self._messages def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Returns: + List of messages converted to standard format for storage. + """ messages = [] for m in self._messages: standard_messages = self.to_standard_messages(m) @@ -154,9 +261,19 @@ class OpenAILLMContext: return messages def set_tool_choice(self, tool_choice: ChatCompletionToolChoiceOptionParam | NotGiven): + """Set the tool choice configuration. + + Args: + tool_choice: Tool selection strategy for the LLM. + """ self._tool_choice = tool_choice def set_tools(self, tools: List[ChatCompletionToolParam] | NotGiven | ToolsSchema = NOT_GIVEN): + """Set the available tools for the LLM. + + Args: + tools: List of tools available to the LLM, or NOT_GIVEN to disable tools. + """ if tools != NOT_GIVEN and isinstance(tools, list) and len(tools) == 0: tools = NOT_GIVEN self._tools = tools @@ -164,6 +281,14 @@ class OpenAILLMContext: def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: str = None ): + """Add a message containing an image frame. + + Args: + format: Image format (e.g., 'RGB', 'RGBA'). + size: Image dimensions as (width, height) tuple. + image: Raw image bytes. + text: Optional text to include with the image. + """ buffer = io.BytesIO() Image.frombytes(format, size, image).save(buffer, format="JPEG") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") @@ -177,10 +302,30 @@ class OpenAILLMContext: self.add_message({"role": "user", "content": content}) def add_audio_frames_message(self, *, audio_frames: list[AudioRawFrame], text: str = None): + """Add a message containing audio frames. + + Args: + audio_frames: List of audio frame objects to include. + text: Optional text to include with the audio. + + Note: + This method is currently a placeholder for future implementation. + """ # todo: implement for OpenAI models and others pass def create_wav_header(self, sample_rate, num_channels, bits_per_sample, data_size): + """Create a WAV file header for audio data. + + Args: + sample_rate: Audio sample rate in Hz. + num_channels: Number of audio channels. + bits_per_sample: Bits per audio sample. + data_size: Size of audio data in bytes. + + Returns: + WAV header as a bytearray. + """ # RIFF chunk descriptor header = bytearray() header.extend(b"RIFF") # ChunkID @@ -206,10 +351,14 @@ class OpenAILLMContext: @dataclass class OpenAILLMContextFrame(Frame): - """Like an LLMMessagesFrame, but with extra context specific to the OpenAI + """Frame containing OpenAI-specific LLM context. + + Like an LLMMessagesFrame, but with extra context specific to the OpenAI API. The context in this message is also mutable, and will be changed by the OpenAIContextAggregator frame processor. + Parameters: + context: The OpenAI LLM context containing messages, tools, and configuration. """ context: OpenAILLMContext diff --git a/src/pipecat/processors/aggregators/sentence.py b/src/pipecat/processors/aggregators/sentence.py index 54aeea16a..5a2bc59dc 100644 --- a/src/pipecat/processors/aggregators/sentence.py +++ b/src/pipecat/processors/aggregators/sentence.py @@ -4,17 +4,28 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Text sentence aggregation processor for Pipecat. + +This module provides a frame processor that accumulates text frames into +complete sentences, only outputting when a sentence-ending pattern is detected. +""" + from pipecat.frames.frames import EndFrame, Frame, InterimTranscriptionFrame, TextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.utils.string import match_endofsentence class SentenceAggregator(FrameProcessor): - """This frame processor aggregates text frames into complete sentences. + """Aggregates text frames into complete sentences. + + This processor accumulates incoming text frames until a sentence-ending + pattern is detected, then outputs the complete sentence as a single frame. + Useful for ensuring downstream processors receive coherent, complete sentences + rather than fragmented text. Frame input/output: TextFrame("Hello,") -> None - TextFrame(" world.") -> TextFrame("Hello world.") + TextFrame(" world.") -> TextFrame("Hello, world.") Doctest: FIXME to work with asyncio >>> import asyncio @@ -29,10 +40,20 @@ class SentenceAggregator(FrameProcessor): """ def __init__(self): + """Initialize the sentence aggregator. + + Sets up internal state for accumulating text frames into complete sentences. + """ super().__init__() self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and aggregate text into complete sentences. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) # We ignore interim description at this point. diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index 8831f7d10..87c9d9f58 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -4,15 +4,39 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""User response aggregation for text frames. + +This module provides an aggregator that collects user responses and outputs +them as TextFrame objects, useful for capturing and processing user input +in conversational pipelines. +""" + from pipecat.frames.frames import TextFrame from pipecat.processors.aggregators.llm_response import LLMUserResponseAggregator class UserResponseAggregator(LLMUserResponseAggregator): + """Aggregates user responses into TextFrame objects. + + This aggregator extends LLMUserResponseAggregator to specifically handle + user input by collecting text responses and outputting them as TextFrame + objects when the aggregation is complete. + """ + def __init__(self, **kwargs): + """Initialize the user response aggregator. + + Args: + **kwargs: Additional arguments passed to parent LLMUserResponseAggregator. + """ super().__init__(**kwargs) async def push_aggregation(self): + """Push the aggregated user response as a TextFrame. + + Creates a TextFrame from the current aggregation if it contains content, + resets the aggregation state, and pushes the frame downstream. + """ if len(self._aggregation) > 0: frame = TextFrame(self._aggregation.strip()) diff --git a/src/pipecat/processors/aggregators/vision_image_frame.py b/src/pipecat/processors/aggregators/vision_image_frame.py index e5e0e41da..ea1848ff1 100644 --- a/src/pipecat/processors/aggregators/vision_image_frame.py +++ b/src/pipecat/processors/aggregators/vision_image_frame.py @@ -4,14 +4,22 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Vision image frame aggregation for Pipecat. + +This module provides frame aggregation functionality to combine text and image +frames into vision frames for multimodal processing. +""" + from pipecat.frames.frames import Frame, InputImageRawFrame, TextFrame, VisionImageRawFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class VisionImageFrameAggregator(FrameProcessor): - """This aggregator waits for a consecutive TextFrame and an - InputImageRawFrame. After the InputImageRawFrame arrives it will output a - VisionImageRawFrame. + """Aggregates consecutive text and image frames into vision frames. + + This aggregator waits for a consecutive TextFrame and an InputImageRawFrame. + After the InputImageRawFrame arrives it will output a VisionImageRawFrame + combining both the text and image data for multimodal processing. >>> from pipecat.frames.frames import ImageFrame @@ -23,14 +31,27 @@ class VisionImageFrameAggregator(FrameProcessor): >>> asyncio.run(print_frames(aggregator, TextFrame("What do you see?"))) >>> asyncio.run(print_frames(aggregator, ImageFrame(image=bytes([]), size=(0, 0)))) VisionImageFrame, text: What do you see?, image size: 0x0, buffer size: 0 B - """ def __init__(self): + """Initialize the vision image frame aggregator. + + The aggregator starts with no cached text, waiting for the first + TextFrame to arrive before it can create vision frames. + """ super().__init__() self._describe_text = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and aggregate text with images. + + Caches TextFrames and combines them with subsequent InputImageRawFrames + to create VisionImageRawFrames. Other frames are passed through unchanged. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, TextFrame): diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py index 142d9cb47..8e03a989a 100644 --- a/src/pipecat/processors/async_generator.py +++ b/src/pipecat/processors/async_generator.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Async generator processor for frame serialization and streaming.""" + import asyncio from typing import Any, AsyncGenerator @@ -17,12 +19,32 @@ from pipecat.serializers.base_serializer import FrameSerializer class AsyncGeneratorProcessor(FrameProcessor): + """A frame processor that serializes frames and provides them via async generator. + + This processor passes frames through unchanged while simultaneously serializing + them and making the serialized data available through an async generator interface. + Useful for streaming frame data to external consumers while maintaining the + normal frame processing pipeline. + """ + def __init__(self, *, serializer: FrameSerializer, **kwargs): + """Initialize the async generator processor. + + Args: + serializer: The frame serializer to use for converting frames to data. + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) self._serializer = serializer self._data_queue = asyncio.Queue() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames by passing them through and queuing serialized data. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) await self.push_frame(frame, direction) @@ -35,6 +57,12 @@ class AsyncGeneratorProcessor(FrameProcessor): await self._data_queue.put(data) async def generator(self) -> AsyncGenerator[Any, None]: + """Generate serialized frame data asynchronously. + + Yields: + Serialized frame data from the internal queue until a termination + signal (None) is received. + """ running = True while running: data = await self._data_queue.get() diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 13d5a84bc..f48d2d6a8 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Audio buffer processor for managing and synchronizing audio streams. + +This module provides an AudioBufferProcessor that handles buffering and synchronization +of audio from both user input and bot output sources, with support for various audio +configurations and event-driven processing. +""" + import time from typing import Optional @@ -37,12 +44,6 @@ class AudioBufferProcessor(FrameProcessor): on_user_turn_audio_data: Triggered when user turn has ended, providing that user turn's audio on_bot_turn_audio_data: Triggered when bot turn has ended, providing that bot turn's audio - Args: - sample_rate (Optional[int]): Desired output sample rate. If None, uses source rate - num_channels (int): Number of channels (1 for mono, 2 for stereo). Defaults to 1 - buffer_size (int): Size of buffer before triggering events. 0 for no buffering - enable_turn_audio (bool): Whether turn audio event handlers should be triggered - Audio handling: - Mono output (num_channels=1): User and bot audio are mixed - Stereo output (num_channels=2): User audio on left, bot audio on right @@ -61,6 +62,16 @@ class AudioBufferProcessor(FrameProcessor): enable_turn_audio: bool = False, **kwargs, ): + """Initialize the audio buffer processor. + + Args: + sample_rate: Desired output sample rate. If None, uses source rate. + num_channels: Number of channels (1 for mono, 2 for stereo). Defaults to 1. + buffer_size: Size of buffer before triggering events. 0 for no buffering. + user_continuous_stream: Deprecated parameter for backwards compatibility. + enable_turn_audio: Whether turn audio event handlers should be triggered. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._init_sample_rate = sample_rate self._sample_rate = 0 @@ -105,7 +116,7 @@ class AudioBufferProcessor(FrameProcessor): """Current sample rate of the audio processor. Returns: - int: The sample rate in Hz + The sample rate in Hz. """ return self._sample_rate @@ -114,7 +125,7 @@ class AudioBufferProcessor(FrameProcessor): """Number of channels in the audio output. Returns: - int: Number of channels (1 for mono, 2 for stereo) + Number of channels (1 for mono, 2 for stereo). """ return self._num_channels @@ -122,7 +133,7 @@ class AudioBufferProcessor(FrameProcessor): """Check if both user and bot audio buffers contain data. Returns: - bool: True if both buffers contain audio data + True if both buffers contain audio data. """ return self._buffer_has_audio(self._user_audio_buffer) and self._buffer_has_audio( self._bot_audio_buffer @@ -135,7 +146,7 @@ class AudioBufferProcessor(FrameProcessor): on the left channel and bot audio on the right channel. Returns: - bytes: Mixed audio data + Mixed audio data as bytes. """ if self._num_channels == 1: return mix_audio(bytes(self._user_audio_buffer), bytes(self._bot_audio_buffer)) @@ -163,7 +174,12 @@ class AudioBufferProcessor(FrameProcessor): self._recording = False async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming audio frames and manage audio buffers.""" + """Process incoming audio frames and manage audio buffers. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) # Update output sample rate if necessary. @@ -181,10 +197,12 @@ class AudioBufferProcessor(FrameProcessor): await self.push_frame(frame, direction) def _update_sample_rate(self, frame: StartFrame): + """Update the sample rate from the start frame.""" self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate self._audio_buffer_size_1s = self._sample_rate * 2 async def _process_recording(self, frame: Frame): + """Process audio frames for recording.""" if isinstance(frame, InputAudioRawFrame): # Add silence if we need to. silence = self._compute_silence(self._last_user_frame_at) @@ -208,6 +226,7 @@ class AudioBufferProcessor(FrameProcessor): await self._call_on_audio_data_handler() async def _process_turn_recording(self, frame: Frame): + """Process frames for turn-based audio recording.""" if isinstance(frame, UserStartedSpeakingFrame): self._user_speaking = True elif isinstance(frame, UserStoppedSpeakingFrame): @@ -242,6 +261,7 @@ class AudioBufferProcessor(FrameProcessor): self._bot_turn_audio_buffer += resampled async def _call_on_audio_data_handler(self): + """Call the audio data event handlers with buffered audio.""" if not self.has_audio() or not self._recording: return @@ -263,23 +283,28 @@ class AudioBufferProcessor(FrameProcessor): self._reset_audio_buffers() def _buffer_has_audio(self, buffer: bytearray) -> bool: + """Check if a buffer contains audio data.""" return buffer is not None and len(buffer) > 0 def _reset_recording(self): + """Reset recording state and buffers.""" self._reset_audio_buffers() self._last_user_frame_at = time.time() self._last_bot_frame_at = time.time() def _reset_audio_buffers(self): + """Reset all audio buffers to empty state.""" self._user_audio_buffer = bytearray() self._bot_audio_buffer = bytearray() self._user_turn_audio_buffer = bytearray() self._bot_turn_audio_buffer = bytearray() async def _resample_audio(self, frame: AudioRawFrame) -> bytes: + """Resample audio frame to the target sample rate.""" return await self._resampler.resample(frame.audio, frame.sample_rate, self._sample_rate) def _compute_silence(self, from_time: float) -> bytes: + """Compute silence to insert based on time gap.""" quiet_time = time.time() - from_time # We should get audio frames very frequently. We introduce silence only # if there's a big enough gap of 1s. diff --git a/src/pipecat/processors/consumer_processor.py b/src/pipecat/processors/consumer_processor.py index 121dd7712..277cef2cd 100644 --- a/src/pipecat/processors/consumer_processor.py +++ b/src/pipecat/processors/consumer_processor.py @@ -4,20 +4,23 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Consumer processor for consuming frames from ProducerProcessor queues.""" + import asyncio from typing import Awaitable, Callable, Optional from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.producer_processor import ProducerProcessor, identity_transformer +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue class ConsumerProcessor(FrameProcessor): - """This class passes-through frames and also consumes frames from a - producer's queue. When a frame from a producer queue is received it will be - pushed to the specified direction. The frames can be transformed into a - different type of frame before being pushed. + """Frame processor that consumes frames from a ProducerProcessor's queue. + This processor passes through frames normally while also consuming frames + from a ProducerProcessor's queue. When frames are received from the producer + queue, they are optionally transformed and pushed in the specified direction. """ def __init__( @@ -28,13 +31,27 @@ class ConsumerProcessor(FrameProcessor): direction: FrameDirection = FrameDirection.DOWNSTREAM, **kwargs, ): + """Initialize the consumer processor. + + Args: + producer: The producer processor to consume frames from. + transformer: Function to transform frames before pushing. Defaults to identity_transformer. + direction: Direction to push consumed frames. Defaults to DOWNSTREAM. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._transformer = transformer self._direction = direction - self._queue: asyncio.Queue = producer.add_consumer() + self._producer = producer self._consumer_task: Optional[asyncio.Task] = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle lifecycle events. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -47,21 +64,24 @@ class ConsumerProcessor(FrameProcessor): await self.push_frame(frame, direction) async def _start(self, _: StartFrame): + """Start the consumer task and register with the producer.""" if not self._consumer_task: + self._queue: WatchdogQueue = self._producer.add_consumer() self._consumer_task = self.create_task(self._consumer_task_handler()) async def _stop(self, _: EndFrame): + """Stop the consumer task.""" if self._consumer_task: await self.cancel_task(self._consumer_task) async def _cancel(self, _: CancelFrame): + """Cancel the consumer task.""" if self._consumer_task: await self.cancel_task(self._consumer_task) async def _consumer_task_handler(self): + """Handle consuming frames from the producer queue.""" while True: frame = await self._queue.get() - self.start_watchdog() new_frame = await self._transformer(frame) await self.push_frame(new_frame, self._direction) - self.reset_watchdog() diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py index 43e2dab95..b2084e4df 100644 --- a/src/pipecat/processors/filters/frame_filter.py +++ b/src/pipecat/processors/filters/frame_filter.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame filtering processor for the Pipecat framework.""" + from typing import Tuple, Type from pipecat.frames.frames import EndFrame, Frame, SystemFrame @@ -11,7 +13,21 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class FrameFilter(FrameProcessor): + """A frame processor that filters frames based on their types. + + This processor acts as a selective gate in the pipeline, allowing only + frames of specified types to pass through. System and end frames are + automatically allowed to pass through to maintain pipeline integrity. + """ + def __init__(self, types: Tuple[Type[Frame], ...]): + """Initialize the frame filter. + + Args: + types: Tuple of frame types that should be allowed to pass through + the filter. All other frame types (except SystemFrame and + EndFrame) will be blocked. + """ super().__init__() self._types = types @@ -20,12 +36,19 @@ class FrameFilter(FrameProcessor): # def _should_passthrough_frame(self, frame): + """Determine if a frame should pass through the filter.""" if isinstance(frame, self._types): return True return isinstance(frame, (EndFrame, SystemFrame)) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process an incoming frame and conditionally pass it through. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if self._should_passthrough_frame(frame): diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index a8fa5b2af..e663b81f4 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Function-based frame filtering for Pipecat pipelines. + +This module provides a processor that filters frames based on a custom function, +allowing for flexible frame filtering logic in processing pipelines. +""" + from typing import Awaitable, Callable from pipecat.frames.frames import EndFrame, Frame, SystemFrame @@ -11,11 +17,26 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class FunctionFilter(FrameProcessor): + """A frame processor that filters frames using a custom function. + + This processor allows frames to pass through based on the result of a + user-provided filter function. System and end frames always pass through + regardless of the filter result. + """ + def __init__( self, filter: Callable[[Frame], Awaitable[bool]], direction: FrameDirection = FrameDirection.DOWNSTREAM, ): + """Initialize the function filter. + + Args: + filter: An async function that takes a Frame and returns True if the + frame should pass through, False otherwise. + direction: The direction to apply filtering. Only frames moving in + this direction will be filtered. Defaults to DOWNSTREAM. + """ super().__init__() self._filter = filter self._direction = direction @@ -27,9 +48,18 @@ class FunctionFilter(FrameProcessor): # Ignore system frames, end frames and frames that are not following the # direction of this gate def _should_passthrough_frame(self, frame, direction): + """Check if a frame should pass through without filtering.""" + # Ignore system frames, end frames and frames that are not following the + # direction of this gate return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process a frame through the filter. + + Args: + frame: The frame to process. + direction: The direction the frame is moving in the pipeline. + """ await super().process_frame(frame, direction) passthrough = self._should_passthrough_frame(frame, direction) diff --git a/src/pipecat/processors/filters/identity_filter.py b/src/pipecat/processors/filters/identity_filter.py index e0bfebcf5..f3999b59c 100644 --- a/src/pipecat/processors/filters/identity_filter.py +++ b/src/pipecat/processors/filters/identity_filter.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Identity filter for transparent frame passthrough. + +This module provides a simple passthrough filter that forwards all frames +without modification, useful for testing and pipeline composition. +""" + from pipecat.frames.frames import Frame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -14,10 +20,14 @@ class IdentityFilter(FrameProcessor): This filter acts as a transparent passthrough, allowing all frames to flow through unchanged. It can be useful when testing `ParallelPipeline` to create pipelines that pass through frames (no frames should be repeated). - """ def __init__(self, **kwargs): + """Initialize the identity filter. + + Args: + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) # @@ -25,6 +35,11 @@ class IdentityFilter(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process an incoming frame by passing it through unchanged.""" + """Process an incoming frame by passing it through unchanged. + + Args: + frame: The frame to process and forward. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/filters/null_filter.py b/src/pipecat/processors/filters/null_filter.py index 8e6a6dda8..1b6670aef 100644 --- a/src/pipecat/processors/filters/null_filter.py +++ b/src/pipecat/processors/filters/null_filter.py @@ -4,14 +4,31 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Null filter processor for blocking frame transmission. + +This module provides a frame processor that blocks all frames except +system and end frames, useful for testing or temporarily stopping +frame flow in a pipeline. +""" + from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class NullFilter(FrameProcessor): - """This filter doesn't allow passing any frames up or downstream.""" + """A filter that blocks all frames except system and end frames. + + This processor acts as a null filter, preventing frames from passing + through the pipeline while still allowing essential system and end + frames to maintain proper pipeline operation. + """ def __init__(self, **kwargs): + """Initialize the null filter. + + Args: + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) # @@ -19,6 +36,12 @@ class NullFilter(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames, only allowing system and end frames through. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, (SystemFrame, EndFrame)): diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index e85a3e581..700313d31 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -39,12 +39,17 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class STTMuteStrategy(Enum): """Strategies determining when STT should be muted. - Attributes: - FIRST_SPEECH: Mute only during first detected bot speech - MUTE_UNTIL_FIRST_BOT_COMPLETE: Start muted and remain muted until first bot speech completes - FUNCTION_CALL: Mute during function calls - ALWAYS: Mute during all bot speech - CUSTOM: Allow custom logic via callback + Each strategy defines different conditions under which speech-to-text + processing should be temporarily disabled to prevent unwanted audio + processing during specific conversation states. + + Parameters: + FIRST_SPEECH: Mute STT until the first bot speech is detected. + MUTE_UNTIL_FIRST_BOT_COMPLETE: Mute STT until the first bot completes speaking, + regardless of whether it is the first speech. + FUNCTION_CALL: Mute STT during function calls to prevent interruptions. + ALWAYS: Always mute STT when the bot is speaking. + CUSTOM: Use a custom callback to determine muting logic dynamically. """ FIRST_SPEECH = "first_speech" @@ -58,10 +63,15 @@ class STTMuteStrategy(Enum): class STTMuteConfig: """Configuration for STT muting behavior. - Args: - strategies: Set of muting strategies to apply + Defines which muting strategies to apply and provides optional custom + callback for advanced muting logic. Multiple strategies can be combined + to create sophisticated muting behavior. + + Parameters: + strategies: Set of muting strategies to apply simultaneously. should_mute_callback: Optional callback for custom muting logic. - Only required when using STTMuteStrategy.CUSTOM + Only required when using STTMuteStrategy.CUSTOM. Called with + the STTMuteFilter instance to determine muting state. Note: MUTE_UNTIL_FIRST_BOT_COMPLETE and FIRST_SPEECH strategies should not be used together @@ -69,10 +79,14 @@ class STTMuteConfig: """ strategies: set[STTMuteStrategy] - # Optional callback for custom muting logic should_mute_callback: Optional[Callable[["STTMuteFilter"], Awaitable[bool]]] = None def __post_init__(self): + """Validate configuration after initialization. + + Raises: + ValueError: If incompatible strategies are used together. + """ if ( STTMuteStrategy.MUTE_UNTIL_FIRST_BOT_COMPLETE in self.strategies and STTMuteStrategy.FIRST_SPEECH in self.strategies @@ -86,15 +100,18 @@ class STTMuteFilter(FrameProcessor): """A processor that handles STT muting and interruption control. This processor combines STT muting and interruption control as a coordinated - feature. When STT is muted, interruptions are automatically disabled. - - Args: - config: Configuration specifying muting strategies - stt_service: STT service instance (deprecated, will be removed in future version) - **kwargs: Additional arguments passed to parent class + feature. When STT is muted, interruptions are automatically disabled by + suppressing VAD-related frames. This prevents unwanted speech detection + during bot speech, function calls, or other specified conditions. """ def __init__(self, *, config: STTMuteConfig, **kwargs): + """Initialize the STT mute filter. + + Args: + config: Configuration specifying muting strategies and behavior. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._config = config self._first_speech_handled = False @@ -104,18 +121,22 @@ class STTMuteFilter(FrameProcessor): @property def is_muted(self) -> bool: - """Returns whether STT is currently muted.""" + """Check if STT is currently muted. + + Returns: + True if STT is currently muted and audio frames are being suppressed. + """ return self._is_muted async def _handle_mute_state(self, should_mute: bool): - """Handles both STT muting and interruption control.""" + """Handle STT muting and interruption control state changes.""" if should_mute != self.is_muted: logger.debug(f"STTMuteFilter {'muting' if should_mute else 'unmuting'}") self._is_muted = should_mute await self.push_frame(STTMuteFrame(mute=should_mute)) async def _should_mute(self) -> bool: - """Determines if STT should be muted based on current state and strategy.""" + """Determine if STT should be muted based on current state and strategies.""" for strategy in self._config.strategies: match strategy: case STTMuteStrategy.FUNCTION_CALL: @@ -144,7 +165,16 @@ class STTMuteFilter(FrameProcessor): return False async def process_frame(self, frame: Frame, direction: FrameDirection): - """Processes incoming frames and manages muting state.""" + """Process incoming frames and manage muting state. + + Monitors conversation state through frame types and applies muting + strategies accordingly. Suppresses VAD-related frames when muted + while allowing other frames to pass through. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) # Determine if we need to change mute state based on frame type diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index d8661eae1..1f01d0a1b 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Wake phrase detection filter for Pipecat transcription processing. + +This module provides a frame processor that filters transcription frames, +only allowing them through after wake phrases have been detected. Includes +keepalive functionality to maintain conversation flow after wake detection. +""" + import re import time from enum import Enum @@ -16,23 +23,53 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class WakeCheckFilter(FrameProcessor): - """This filter looks for wake phrases in the transcription frames and only passes through frames - after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief - period of continued conversation after a wake phrase has been detected. + """Frame processor that filters transcription frames based on wake phrase detection. + + This filter monitors transcription frames for configured wake phrases and only + passes frames through after a wake phrase has been detected. Maintains a + keepalive timeout to allow continued conversation after wake detection. """ class WakeState(Enum): + """Enumeration of wake detection states. + + Parameters: + IDLE: No wake phrase detected, filtering active. + AWAKE: Wake phrase detected, allowing frames through. + """ + IDLE = 1 AWAKE = 2 class ParticipantState: + """State tracking for individual participants. + + Parameters: + participant_id: Unique identifier for the participant. + state: Current wake state (IDLE or AWAKE). + wake_timer: Timestamp of last wake phrase detection. + accumulator: Accumulated text for wake phrase matching. + """ + def __init__(self, participant_id: str): + """Initialize participant state. + + Args: + participant_id: Unique identifier for the participant. + """ self.participant_id = participant_id self.state = WakeCheckFilter.WakeState.IDLE self.wake_timer = 0.0 self.accumulator = "" def __init__(self, wake_phrases: List[str], keepalive_timeout: float = 3): + """Initialize the wake phrase filter. + + Args: + wake_phrases: List of wake phrases to detect in transcriptions. + keepalive_timeout: Duration in seconds to keep passing frames after + wake detection. Defaults to 3 seconds. + """ super().__init__() self._participant_states = {} self._keepalive_timeout = keepalive_timeout @@ -44,6 +81,12 @@ class WakeCheckFilter(FrameProcessor): self._wake_patterns.append(pattern) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames, filtering transcriptions based on wake detection. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) try: diff --git a/src/pipecat/processors/filters/wake_notifier_filter.py b/src/pipecat/processors/filters/wake_notifier_filter.py index c7e6ed27a..c30f3b5d3 100644 --- a/src/pipecat/processors/filters/wake_notifier_filter.py +++ b/src/pipecat/processors/filters/wake_notifier_filter.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Wake notifier filter for conditional frame-based notifications.""" + from typing import Awaitable, Callable, Tuple, Type from pipecat.frames.frames import Frame @@ -12,10 +14,11 @@ from pipecat.sync.base_notifier import BaseNotifier class WakeNotifierFilter(FrameProcessor): - """This processor expects a list of frame types and will execute a given - callback predicate when a frame of any of those type is being processed. If - the callback returns true the notifier will be notified. + """Frame processor that conditionally triggers notifications based on frame types and filters. + This processor monitors frames of specified types and executes a callback predicate + when such frames are processed. If the callback returns True, the associated + notifier is triggered, allowing for conditional wake-up or notification scenarios. """ def __init__( @@ -26,12 +29,27 @@ class WakeNotifierFilter(FrameProcessor): filter: Callable[[Frame], Awaitable[bool]], **kwargs, ): + """Initialize the wake notifier filter. + + Args: + notifier: The notifier to trigger when conditions are met. + types: Tuple of frame types to monitor for potential notifications. + filter: Async callback that determines whether to trigger notification. + Should return True to trigger notification, False otherwise. + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) self._notifier = notifier self._types = types self._filter = filter async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and conditionally trigger notifications. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, self._types) and await self._filter(frame): diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index bee5ce91c..4105f4179 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame processing pipeline infrastructure for Pipecat. + +This module provides the core frame processing system that enables building +audio/video processing pipelines. It includes frame processors, pipeline +management, and frame flow control mechanisms. +""" + import asyncio from dataclasses import dataclass from enum import Enum @@ -29,42 +36,83 @@ from pipecat.frames.frames import ( from pipecat.metrics.metrics import LLMTokenUsage, MetricsData from pipecat.observers.base_observer import BaseObserver, FramePushed from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMetrics -from pipecat.utils.asyncio import BaseTaskManager +from pipecat.utils.asyncio.task_manager import BaseTaskManager +from pipecat.utils.asyncio.watchdog_event import WatchdogEvent +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue from pipecat.utils.base_object import BaseObject class FrameDirection(Enum): + """Direction of frame flow in the processing pipeline. + + Parameters: + DOWNSTREAM: Frames flowing from input to output. + UPSTREAM: Frames flowing back from output to input. + """ + DOWNSTREAM = 1 UPSTREAM = 2 @dataclass class FrameProcessorSetup: + """Configuration parameters for frame processor initialization. + + Parameters: + clock: The clock instance for timing operations. + task_manager: The task manager for handling async operations. + observer: Optional observer for monitoring frame processing events. + watchdog_timers_enabled: Whether to enable watchdog timers by default. + """ + clock: BaseClock task_manager: BaseTaskManager observer: Optional[BaseObserver] = None + watchdog_timers_enabled: bool = False class FrameProcessor(BaseObject): + """Base class for all frame processors in the pipeline. + + Frame processors are the building blocks of Pipecat pipelines. They receive + frames, process them, and pass them to the next processor in the chain. + Each processor runs in its own task and can be linked to form complex + processing pipelines. + """ + def __init__( self, *, name: Optional[str] = None, - metrics: Optional[FrameProcessorMetrics] = None, enable_watchdog_logging: Optional[bool] = None, + enable_watchdog_timers: Optional[bool] = None, + metrics: Optional[FrameProcessorMetrics] = None, watchdog_timeout_secs: Optional[float] = None, **kwargs, ): + """Initialize the frame processor. + + Args: + name: Optional name for this processor instance. + enable_watchdog_logging: Whether to enable watchdog logging for tasks. + enable_watchdog_timers: Whether to enable watchdog timers for tasks. + metrics: Optional metrics collector for this processor. + watchdog_timeout_secs: Timeout in seconds for watchdog operations. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(name=name) self._parent: Optional["FrameProcessor"] = None self._prev: Optional["FrameProcessor"] = None self._next: Optional["FrameProcessor"] = None + # Enable watchdog timers for all tasks created by this frame processor. + self._enable_watchdog_timers = enable_watchdog_timers + # Enable watchdog logging for all tasks created by this frame processor. self._enable_watchdog_logging = enable_watchdog_logging # Allow this frame processor to control their tasks timeout. - self._watchdog_timeout = watchdog_timeout_secs + self._watchdog_timeout_secs = watchdog_timeout_secs # Clock self._clock: Optional[BaseClock] = None @@ -101,7 +149,7 @@ class FrameProcessor(BaseObject): # is called. To resume processing frames we need to call # `resume_processing_frames()` which will wake up the event. self.__should_block_frames = False - self.__input_event = asyncio.Event() + self.__input_event = None self.__input_frame_task: Optional[asyncio.Task] = None # Every processor in Pipecat should only output frames from a single @@ -111,71 +159,145 @@ class FrameProcessor(BaseObject): @property def id(self) -> int: + """Get the unique identifier for this processor. + + Returns: + The unique integer ID of this processor. + """ return self._id @property def name(self) -> str: + """Get the name of this processor. + + Returns: + The name of this processor instance. + """ return self._name @property def interruptions_allowed(self): + """Check if interruptions are allowed for this processor. + + Returns: + True if interruptions are allowed. + """ return self._allow_interruptions @property def metrics_enabled(self): + """Check if metrics collection is enabled. + + Returns: + True if metrics collection is enabled. + """ return self._enable_metrics @property def usage_metrics_enabled(self): + """Check if usage metrics collection is enabled. + + Returns: + True if usage metrics collection is enabled. + """ return self._enable_usage_metrics @property def report_only_initial_ttfb(self): + """Check if only initial TTFB should be reported. + + Returns: + True if only initial time-to-first-byte should be reported. + """ return self._report_only_initial_ttfb @property def interruption_strategies(self) -> Sequence[BaseInterruptionStrategy]: + """Get the interruption strategies for this processor. + + Returns: + Sequence of interruption strategies. + """ return self._interruption_strategies + @property + def task_manager(self) -> BaseTaskManager: + """Get the task manager for this processor. + + Returns: + The task manager instance. + + Raises: + Exception: If the task manager is not initialized. + """ + if not self._task_manager: + raise Exception(f"{self} TaskManager is still not initialized.") + return self._task_manager + def can_generate_metrics(self) -> bool: + """Check if this processor can generate metrics. + + Returns: + True if this processor can generate metrics. + """ return False def set_core_metrics_data(self, data: MetricsData): + """Set core metrics data for this processor. + + Args: + data: The metrics data to set. + """ self._metrics.set_core_metrics_data(data) async def start_ttfb_metrics(self): + """Start time-to-first-byte metrics collection.""" if self.can_generate_metrics() and self.metrics_enabled: await self._metrics.start_ttfb_metrics(self._report_only_initial_ttfb) async def stop_ttfb_metrics(self): + """Stop time-to-first-byte metrics collection and push results.""" if self.can_generate_metrics() and self.metrics_enabled: frame = await self._metrics.stop_ttfb_metrics() if frame: await self.push_frame(frame) async def start_processing_metrics(self): + """Start processing metrics collection.""" if self.can_generate_metrics() and self.metrics_enabled: await self._metrics.start_processing_metrics() async def stop_processing_metrics(self): + """Stop processing metrics collection and push results.""" if self.can_generate_metrics() and self.metrics_enabled: frame = await self._metrics.stop_processing_metrics() if frame: await self.push_frame(frame) async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): + """Start LLM usage metrics collection. + + Args: + tokens: Token usage information for the LLM. + """ if self.can_generate_metrics() and self.usage_metrics_enabled: frame = await self._metrics.start_llm_usage_metrics(tokens) if frame: await self.push_frame(frame) async def start_tts_usage_metrics(self, text: str): + """Start TTS usage metrics collection. + + Args: + text: The text being processed by TTS. + """ if self.can_generate_metrics() and self.usage_metrics_enabled: frame = await self._metrics.start_tts_usage_metrics(text) if frame: await self.push_frame(frame) async def stop_all_metrics(self): + """Stop all active metrics collection.""" await self.stop_ttfb_metrics() await self.stop_processing_metrics() @@ -185,13 +307,26 @@ class FrameProcessor(BaseObject): name: Optional[str] = None, *, enable_watchdog_logging: Optional[bool] = None, + enable_watchdog_timers: Optional[bool] = None, watchdog_timeout_secs: Optional[float] = None, ) -> asyncio.Task: + """Create a new task managed by this processor. + + Args: + coroutine: The coroutine to run in the task. + name: Optional name for the task. + enable_watchdog_logging: Whether to enable watchdog logging. + enable_watchdog_timers: Whether to enable watchdog timers. + watchdog_timeout_secs: Timeout in seconds for watchdog operations. + + Returns: + The created asyncio task. + """ if name: name = f"{self}::{name}" else: name = f"{self}::{coroutine.cr_code.co_name}" - return self.get_task_manager().create_task( + return self.task_manager.create_task( coroutine, name, enable_watchdog_logging=( @@ -199,31 +334,55 @@ class FrameProcessor(BaseObject): if enable_watchdog_logging else self._enable_watchdog_logging ), + enable_watchdog_timers=( + enable_watchdog_timers if enable_watchdog_timers else self._enable_watchdog_timers + ), watchdog_timeout=( - watchdog_timeout_secs if watchdog_timeout_secs else self._watchdog_timeout + watchdog_timeout_secs if watchdog_timeout_secs else self._watchdog_timeout_secs ), ) async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): - await self.get_task_manager().cancel_task(task, timeout) + """Cancel a task managed by this processor. + + Args: + task: The task to cancel. + timeout: Optional timeout for task cancellation. + """ + await self.task_manager.cancel_task(task, timeout) async def wait_for_task(self, task: asyncio.Task, timeout: Optional[float] = None): - await self.get_task_manager().wait_for_task(task, timeout) + """Wait for a task to complete. - def start_watchdog(self): - self.get_task_manager().start_watchdog(asyncio.current_task()) + Args: + task: The task to wait for. + timeout: Optional timeout for waiting. + """ + await self.task_manager.wait_for_task(task, timeout) def reset_watchdog(self): - self.get_task_manager().reset_watchdog(asyncio.current_task()) + """Reset the watchdog timer for the current task.""" + self.task_manager.task_reset_watchdog() async def setup(self, setup: FrameProcessorSetup): + """Set up the processor with required components. + + Args: + setup: Configuration object containing setup parameters. + """ self._clock = setup.clock self._task_manager = setup.task_manager self._observer = setup.observer + self._watchdog_timers_enabled = ( + self._enable_watchdog_timers + if self._enable_watchdog_timers + else setup.watchdog_timers_enabled + ) if self._metrics is not None: await self._metrics.setup(self._task_manager) async def cleanup(self): + """Clean up processor resources.""" await super().cleanup() await self.__cancel_input_task() await self.__cancel_push_task() @@ -231,29 +390,52 @@ class FrameProcessor(BaseObject): await self._metrics.cleanup() def link(self, processor: "FrameProcessor"): + """Link this processor to the next processor in the pipeline. + + Args: + processor: The processor to link to. + """ self._next = processor processor._prev = self logger.debug(f"Linking {self} -> {self._next}") def get_event_loop(self) -> asyncio.AbstractEventLoop: - return self.get_task_manager().get_event_loop() + """Get the event loop used by this processor. + + Returns: + The asyncio event loop. + """ + return self.task_manager.get_event_loop() def set_parent(self, parent: "FrameProcessor"): + """Set the parent processor for this processor. + + Args: + parent: The parent processor. + """ self._parent = parent def get_parent(self) -> Optional["FrameProcessor"]: + """Get the parent processor. + + Returns: + The parent processor, or None if no parent is set. + """ return self._parent def get_clock(self) -> BaseClock: + """Get the clock used by this processor. + + Returns: + The clock instance. + + Raises: + Exception: If the clock is not initialized. + """ if not self._clock: raise Exception(f"{self} Clock is still not initialized.") return self._clock - def get_task_manager(self) -> BaseTaskManager: - if not self._task_manager: - raise Exception(f"{self} TaskManager is still not initialized.") - return self._task_manager - async def queue_frame( self, frame: Frame, @@ -262,6 +444,13 @@ class FrameProcessor(BaseObject): Callable[["FrameProcessor", Frame, FrameDirection], Awaitable[None]] ] = None, ): + """Queue a frame for processing. + + Args: + frame: The frame to queue. + direction: The direction of frame flow. + callback: Optional callback to call after processing. + """ # If we are cancelling we don't want to process any other frame. if self._cancelling: return @@ -274,14 +463,23 @@ class FrameProcessor(BaseObject): await self.__input_queue.put((frame, direction, callback)) async def pause_processing_frames(self): + """Pause processing of queued frames.""" logger.trace(f"{self}: pausing frame processing") self.__should_block_frames = True async def resume_processing_frames(self): + """Resume processing of queued frames.""" logger.trace(f"{self}: resuming frame processing") - self.__input_event.set() + if self.__input_event: + self.__input_event.set() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process a frame. + + Args: + frame: The frame to process. + direction: The direction of frame flow. + """ if isinstance(frame, StartFrame): await self.__start(frame) elif isinstance(frame, StartInterruptionFrame): @@ -297,9 +495,20 @@ class FrameProcessor(BaseObject): await self.__resume(frame) async def push_error(self, error: ErrorFrame): + """Push an error frame upstream. + + Args: + error: The error frame to push. + """ await self.push_frame(error, FrameDirection.UPSTREAM) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame to the next processor in the pipeline. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ if not self._check_started(frame): return @@ -309,25 +518,45 @@ class FrameProcessor(BaseObject): await self.__push_queue.put((frame, direction)) async def __start(self, frame: StartFrame): + """Handle the start frame to initialize processor state. + + Args: + frame: The start frame containing initialization parameters. + """ self.__started = True self._allow_interruptions = frame.allow_interruptions self._enable_metrics = frame.enable_metrics self._enable_usage_metrics = frame.enable_usage_metrics - self._report_only_initial_ttfb = frame.report_only_initial_ttfb self._interruption_strategies = frame.interruption_strategies + self._report_only_initial_ttfb = frame.report_only_initial_ttfb self.__create_input_task() self.__create_push_task() async def __cancel(self, frame: CancelFrame): + """Handle the cancel frame to stop processor operation. + + Args: + frame: The cancel frame. + """ self._cancelling = True await self.__cancel_input_task() await self.__cancel_push_task() async def __pause(self, frame: FrameProcessorPauseFrame | FrameProcessorPauseUrgentFrame): + """Handle pause frame to pause processor operation. + + Args: + frame: The pause frame. + """ if frame.processor.name == self.name: await self.pause_processing_frames() async def __resume(self, frame: FrameProcessorResumeFrame | FrameProcessorResumeUrgentFrame): + """Handle resume frame to resume processor operation. + + Args: + frame: The resume frame. + """ if frame.processor.name == self.name: await self.resume_processing_frames() @@ -336,6 +565,7 @@ class FrameProcessor(BaseObject): # async def _start_interruption(self): + """Start handling an interruption by canceling current tasks.""" try: # Cancel the push frame task. This will stop pushing frames downstream. await self.__cancel_push_task() @@ -353,10 +583,17 @@ class FrameProcessor(BaseObject): self.__create_push_task() async def _stop_interruption(self): + """Stop handling an interruption.""" # Nothing to do right now. pass async def __internal_push_frame(self, frame: Frame, direction: FrameDirection): + """Internal method to push frames to adjacent processors. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ try: timestamp = self._clock.get_time() if self._clock else 0 if direction == FrameDirection.DOWNSTREAM and self._next: @@ -389,25 +626,38 @@ class FrameProcessor(BaseObject): await self.push_error(ErrorFrame(str(e))) def _check_started(self, frame: Frame): + """Check if the processor has been started. + + Args: + frame: The frame being processed. + + Returns: + True if the processor has been started. + """ if not self.__started: logger.error(f"{self} Trying to process {frame} but StartFrame not received yet") return self.__started def __create_input_task(self): + """Create the input processing task.""" if not self.__input_frame_task: self.__should_block_frames = False + if not self.__input_event: + self.__input_event = WatchdogEvent(self.task_manager) self.__input_event.clear() - self.__input_queue = asyncio.Queue() + self.__input_queue = WatchdogQueue(self.task_manager) self.__input_frame_task = self.create_task(self.__input_frame_task_handler()) async def __cancel_input_task(self): + """Cancel the input processing task.""" if self.__input_frame_task: await self.cancel_task(self.__input_frame_task) self.__input_frame_task = None async def __input_frame_task_handler(self): + """Handle frames from the input queue.""" while True: - if self.__should_block_frames: + if self.__should_block_frames and self.__input_event: logger.trace(f"{self}: frame processing paused") await self.__input_event.wait() self.__input_event.clear() @@ -416,7 +666,6 @@ class FrameProcessor(BaseObject): (frame, direction, callback) = await self.__input_queue.get() try: - self.start_watchdog() # Process the frame. await self.process_frame(frame, direction) # If this frame has an associated callback, call it now. @@ -427,22 +676,22 @@ class FrameProcessor(BaseObject): await self.push_error(ErrorFrame(str(e))) finally: self.__input_queue.task_done() - self.reset_watchdog() def __create_push_task(self): + """Create the frame pushing task.""" if not self.__push_frame_task: - self.__push_queue = asyncio.Queue() + self.__push_queue = WatchdogQueue(self.task_manager) self.__push_frame_task = self.create_task(self.__push_frame_task_handler()) async def __cancel_push_task(self): + """Cancel the frame pushing task.""" if self.__push_frame_task: await self.cancel_task(self.__push_frame_task) self.__push_frame_task = None async def __push_frame_task_handler(self): + """Handle frames from the push queue.""" while True: (frame, direction) = await self.__push_queue.get() - self.start_watchdog() await self.__internal_push_frame(frame, direction) self.__push_queue.task_done() - self.reset_watchdog() diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index dee197a51..65bf56b70 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Langchain integration processor for Pipecat.""" + from typing import Optional, Union from loguru import logger @@ -26,16 +28,40 @@ except ModuleNotFoundError as e: class LangchainProcessor(FrameProcessor): + """Processor that integrates Langchain runnables with Pipecat's frame pipeline. + + This processor takes LLM message frames, extracts the latest user message, + and processes it through a Langchain runnable chain. The response is streamed + back as text frames with appropriate response markers. + """ + def __init__(self, chain: Runnable, transcript_key: str = "input"): + """Initialize the Langchain processor. + + Args: + chain: The Langchain runnable to use for processing messages. + transcript_key: The key to use when passing input to the chain. + """ super().__init__() self._chain = chain self._transcript_key = transcript_key self._participant_id: Optional[str] = None def set_participant_id(self, participant_id: str): + """Set the participant ID for session tracking. + + Args: + participant_id: The participant ID to use for session configuration. + """ self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle LLM message frames. + + Args: + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, LLMMessagesFrame): @@ -50,6 +76,14 @@ class LangchainProcessor(FrameProcessor): @staticmethod def __get_token_value(text: Union[str, AIMessageChunk]) -> str: + """Extract token value from various text types. + + Args: + text: The text or message chunk to extract value from. + + Returns: + The extracted string value. + """ match text: case str(): return text @@ -59,6 +93,7 @@ class LangchainProcessor(FrameProcessor): return "" async def _ainvoke(self, text: str): + """Invoke the Langchain runnable with the provided text.""" logger.debug(f"Invoking chain with {text}") await self.push_frame(LLMFullResponseStartFrame()) try: diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 29e85e5d7..22d5370f2 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""RTVI (Real-Time Voice Interface) protocol implementation for Pipecat. + +This module provides the RTVI protocol implementation for real-time voice interactions +between clients and AI agents. It includes message handling, action processing, +and frame observation for the RTVI protocol. +""" + import asyncio import base64 from dataclasses import dataclass @@ -67,6 +74,7 @@ from pipecat.services.llm_service import ( from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue from pipecat.utils.string import match_endofsentence RTVI_PROTOCOL_VERSION = "0.3.0" @@ -78,6 +86,12 @@ ActionResult = Union[bool, int, float, str, list, dict] class RTVIServiceOption(BaseModel): + """Configuration option for an RTVI service. + + Defines a configurable option that can be set for an RTVI service, + including its name, type, and handler function. + """ + name: str type: Literal["bool", "number", "string", "array", "object"] handler: Callable[["RTVIProcessor", str, "RTVIServiceOptionConfig"], Awaitable[None]] = Field( @@ -86,11 +100,18 @@ class RTVIServiceOption(BaseModel): class RTVIService(BaseModel): + """An RTVI service definition. + + Represents a service that can be configured and used within the RTVI protocol, + containing a name and list of configurable options. + """ + name: str options: List[RTVIServiceOption] _options_dict: Dict[str, RTVIServiceOption] = PrivateAttr(default={}) def model_post_init(self, __context: Any) -> None: + """Initialize the options dictionary after model creation.""" self._options_dict = {} for option in self.options: self._options_dict[option.name] = option @@ -98,16 +119,32 @@ class RTVIService(BaseModel): class RTVIActionArgumentData(BaseModel): + """Data for an RTVI action argument. + + Contains the name and value of an argument passed to an RTVI action. + """ + name: str value: Any class RTVIActionArgument(BaseModel): + """Definition of an RTVI action argument. + + Specifies the name and expected type of an argument for an RTVI action. + """ + name: str type: Literal["bool", "number", "string", "array", "object"] class RTVIAction(BaseModel): + """An RTVI action definition. + + Represents an action that can be executed within the RTVI protocol, + including its service, name, arguments, and handler function. + """ + service: str action: str arguments: List[RTVIActionArgument] = Field(default_factory=list) @@ -118,6 +155,7 @@ class RTVIAction(BaseModel): _arguments_dict: Dict[str, RTVIActionArgument] = PrivateAttr(default={}) def model_post_init(self, __context: Any) -> None: + """Initialize the arguments dictionary after model creation.""" self._arguments_dict = {} for arg in self.arguments: self._arguments_dict[arg.name] = arg @@ -125,16 +163,31 @@ class RTVIAction(BaseModel): class RTVIServiceOptionConfig(BaseModel): + """Configuration value for an RTVI service option. + + Contains the name and value to set for a specific service option. + """ + name: str value: Any class RTVIServiceConfig(BaseModel): + """Configuration for an RTVI service. + + Contains the service name and list of option configurations to apply. + """ + service: str options: List[RTVIServiceOptionConfig] class RTVIConfig(BaseModel): + """Complete RTVI configuration. + + Contains the full configuration for all RTVI services. + """ + config: List[RTVIServiceConfig] @@ -144,16 +197,31 @@ class RTVIConfig(BaseModel): class RTVIUpdateConfig(BaseModel): + """Request to update RTVI configuration. + + Contains new configuration settings and whether to interrupt the bot. + """ + config: List[RTVIServiceConfig] interrupt: bool = False class RTVIActionRunArgument(BaseModel): + """Argument for running an RTVI action. + + Contains the name and value of an argument to pass to an action. + """ + name: str value: Any class RTVIActionRun(BaseModel): + """Request to run an RTVI action. + + Contains the service, action name, and optional arguments. + """ + service: str action: str arguments: Optional[List[RTVIActionRunArgument]] = None @@ -161,11 +229,23 @@ class RTVIActionRun(BaseModel): @dataclass class RTVIActionFrame(DataFrame): + """Frame containing an RTVI action to execute. + + Parameters: + rtvi_action_run: The action to execute. + message_id: Optional message ID for response correlation. + """ + rtvi_action_run: RTVIActionRun message_id: Optional[str] = None class RTVIMessage(BaseModel): + """Base RTVI message structure. + + Represents the standard format for RTVI protocol messages. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: str id: str @@ -178,10 +258,20 @@ class RTVIMessage(BaseModel): class RTVIErrorResponseData(BaseModel): + """Data for an RTVI error response. + + Contains the error message to send back to the client. + """ + error: str class RTVIErrorResponse(BaseModel): + """RTVI error response message. + + Sent in response to a client request that resulted in an error. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["error-response"] = "error-response" id: str @@ -189,21 +279,41 @@ class RTVIErrorResponse(BaseModel): class RTVIErrorData(BaseModel): + """Data for an RTVI error event. + + Contains error information including whether it's fatal. + """ + error: str fatal: bool class RTVIError(BaseModel): + """RTVI error event message. + + Sent when an error occurs that isn't in response to a specific request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["error"] = "error" data: RTVIErrorData class RTVIDescribeConfigData(BaseModel): + """Data for describing available RTVI configuration. + + Contains the list of available services and their options. + """ + config: List[RTVIService] class RTVIDescribeConfig(BaseModel): + """Message describing available RTVI configuration. + + Sent in response to a describe-config request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["config-available"] = "config-available" id: str @@ -211,10 +321,20 @@ class RTVIDescribeConfig(BaseModel): class RTVIDescribeActionsData(BaseModel): + """Data for describing available RTVI actions. + + Contains the list of available actions that can be executed. + """ + actions: List[RTVIAction] class RTVIDescribeActions(BaseModel): + """Message describing available RTVI actions. + + Sent in response to a describe-actions request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["actions-available"] = "actions-available" id: str @@ -222,6 +342,11 @@ class RTVIDescribeActions(BaseModel): class RTVIConfigResponse(BaseModel): + """Response containing current RTVI configuration. + + Sent in response to a get-config request. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["config"] = "config" id: str @@ -229,10 +354,20 @@ class RTVIConfigResponse(BaseModel): class RTVIActionResponseData(BaseModel): + """Data for an RTVI action response. + + Contains the result of executing an action. + """ + result: ActionResult class RTVIActionResponse(BaseModel): + """Response to an RTVI action execution. + + Sent after successfully executing an action. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["action-response"] = "action-response" id: str @@ -240,11 +375,21 @@ class RTVIActionResponse(BaseModel): class RTVIBotReadyData(BaseModel): + """Data for bot ready notification. + + Contains protocol version and initial configuration. + """ + version: str config: List[RTVIServiceConfig] class RTVIBotReady(BaseModel): + """Message indicating bot is ready for interaction. + + Sent after bot initialization is complete. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-ready"] = "bot-ready" id: str @@ -252,28 +397,53 @@ class RTVIBotReady(BaseModel): class RTVILLMFunctionCallMessageData(BaseModel): + """Data for LLM function call notification. + + Contains function call details including name, ID, and arguments. + """ + function_name: str tool_call_id: str args: Mapping[str, Any] class RTVILLMFunctionCallMessage(BaseModel): + """Message notifying of an LLM function call. + + Sent when the LLM makes a function call. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["llm-function-call"] = "llm-function-call" data: RTVILLMFunctionCallMessageData class RTVILLMFunctionCallStartMessageData(BaseModel): + """Data for LLM function call start notification. + + Contains the function name being called. + """ + function_name: str class RTVILLMFunctionCallStartMessage(BaseModel): + """Message notifying that an LLM function call has started. + + Sent when the LLM begins a function call. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["llm-function-call-start"] = "llm-function-call-start" data: RTVILLMFunctionCallStartMessageData class RTVILLMFunctionCallResultData(BaseModel): + """Data for LLM function call result. + + Contains function call details and result. + """ + function_name: str tool_call_id: str arguments: dict @@ -281,60 +451,103 @@ class RTVILLMFunctionCallResultData(BaseModel): class RTVIBotLLMStartedMessage(BaseModel): + """Message indicating bot LLM processing has started.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-llm-started"] = "bot-llm-started" class RTVIBotLLMStoppedMessage(BaseModel): + """Message indicating bot LLM processing has stopped.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-llm-stopped"] = "bot-llm-stopped" class RTVIBotTTSStartedMessage(BaseModel): + """Message indicating bot TTS processing has started.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-started"] = "bot-tts-started" class RTVIBotTTSStoppedMessage(BaseModel): + """Message indicating bot TTS processing has stopped.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-stopped"] = "bot-tts-stopped" class RTVITextMessageData(BaseModel): + """Data for text-based RTVI messages. + + Contains text content. + """ + text: str class RTVIBotTranscriptionMessage(BaseModel): + """Message containing bot transcription text. + + Sent when the bot's speech is transcribed. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-transcription"] = "bot-transcription" data: RTVITextMessageData class RTVIBotLLMTextMessage(BaseModel): + """Message containing bot LLM text output. + + Sent when the bot's LLM generates text. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-llm-text"] = "bot-llm-text" data: RTVITextMessageData class RTVIBotTTSTextMessage(BaseModel): + """Message containing bot TTS text output. + + Sent when text is being processed by TTS. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-text"] = "bot-tts-text" data: RTVITextMessageData class RTVIAudioMessageData(BaseModel): + """Data for audio-based RTVI messages. + + Contains audio data and metadata. + """ + audio: str sample_rate: int num_channels: int class RTVIBotTTSAudioMessage(BaseModel): + """Message containing bot TTS audio output. + + Sent when the bot's TTS generates audio. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-tts-audio"] = "bot-tts-audio" data: RTVIAudioMessageData class RTVIUserTranscriptionMessageData(BaseModel): + """Data for user transcription messages. + + Contains transcription text and metadata. + """ + text: str user_id: str timestamp: str @@ -342,44 +555,72 @@ class RTVIUserTranscriptionMessageData(BaseModel): class RTVIUserTranscriptionMessage(BaseModel): + """Message containing user transcription. + + Sent when user speech is transcribed. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-transcription"] = "user-transcription" data: RTVIUserTranscriptionMessageData class RTVIUserLLMTextMessage(BaseModel): + """Message containing user text input for LLM. + + Sent when user text is processed by the LLM. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-llm-text"] = "user-llm-text" data: RTVITextMessageData class RTVIUserStartedSpeakingMessage(BaseModel): + """Message indicating user has started speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-started-speaking"] = "user-started-speaking" class RTVIUserStoppedSpeakingMessage(BaseModel): + """Message indicating user has stopped speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["user-stopped-speaking"] = "user-stopped-speaking" class RTVIBotStartedSpeakingMessage(BaseModel): + """Message indicating bot has started speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-started-speaking"] = "bot-started-speaking" class RTVIBotStoppedSpeakingMessage(BaseModel): + """Message indicating bot has stopped speaking.""" + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["bot-stopped-speaking"] = "bot-stopped-speaking" class RTVIMetricsMessage(BaseModel): + """Message containing performance metrics. + + Sent to provide performance and usage metrics. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["metrics"] = "metrics" data: Mapping[str, Any] class RTVIServerMessage(BaseModel): + """Generic server message. + + Used for custom server-to-client messages. + """ + label: RTVIMessageLiteral = RTVI_MESSAGE_LABEL type: Literal["server-message"] = "server-message" data: Any @@ -387,28 +628,32 @@ class RTVIServerMessage(BaseModel): @dataclass class RTVIServerMessageFrame(SystemFrame): - """A frame for sending server messages to the client.""" + """A frame for sending server messages to the client. + + Parameters: + data: The message data to send to the client. + """ data: Any def __str__(self): + """String representation of the RTVI server message frame.""" return f"{self.name}(data: {self.data})" @dataclass class RTVIObserverParams: - """ - Parameters for configuring RTVI Observer behavior. + """Parameters for configuring RTVI Observer behavior. - Attributes: - bot_llm_enabled (bool): Indicates if the bot's LLM messages should be sent. - bot_tts_enabled (bool): Indicates if the bot's TTS messages should be sent. - bot_speaking_enabled (bool): Indicates if the bot's started/stopped speaking messages should be sent. - user_llm_enabled (bool): Indicates if the user's LLM input messages should be sent. - user_speaking_enabled (bool): Indicates if the user's started/stopped speaking messages should be sent. - user_transcription_enabled (bool): Indicates if user's transcription messages should be sent. - metrics_enabled (bool): Indicates if metrics messages should be sent. - errors_enabled (bool): Indicates if errors messages should be sent. + Parameters: + bot_llm_enabled: Indicates if the bot's LLM messages should be sent. + bot_tts_enabled: Indicates if the bot's TTS messages should be sent. + bot_speaking_enabled: Indicates if the bot's started/stopped speaking messages should be sent. + user_llm_enabled: Indicates if the user's LLM input messages should be sent. + user_speaking_enabled: Indicates if the user's started/stopped speaking messages should be sent. + user_transcription_enabled: Indicates if user's transcription messages should be sent. + metrics_enabled: Indicates if metrics messages should be sent. + errors_enabled: Indicates if errors messages should be sent. """ bot_llm_enabled: bool = True @@ -431,15 +676,18 @@ class RTVIObserver(BaseObserver): Note: This observer only handles outgoing messages. Incoming RTVI client messages are handled by the RTVIProcessor. - - Args: - rtvi (RTVIProcessor): The RTVI processor to push frames to. - params (RTVIObserverParams): Settings to enable/disable specific messages. """ def __init__( self, rtvi: "RTVIProcessor", *, params: Optional[RTVIObserverParams] = None, **kwargs ): + """Initialize the RTVI observer. + + Args: + rtvi: The RTVI processor to push frames to. + params: Settings to enable/disable specific messages. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._rtvi = rtvi self._params = params or RTVIObserverParams() @@ -451,11 +699,7 @@ class RTVIObserver(BaseObserver): """Process a frame being pushed through the pipeline. Args: - src: Source processor pushing the frame - dst: Destination processor receiving the frame - frame: The frame being pushed - direction: Direction of frame flow in pipeline - timestamp: Time when frame was pushed + data: Frame push event data containing source, frame, direction, and timestamp. """ src = data.source frame = data.frame @@ -516,13 +760,14 @@ class RTVIObserver(BaseObserver): """Push an urgent transport message to the RTVI processor. Args: - model: The message model to send - exclude_none: Whether to exclude None values from the model dump + model: The message model to send. + exclude_none: Whether to exclude None values from the model dump. """ frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) await self._rtvi.push_frame(frame) async def _push_bot_transcription(self): + """Push accumulated bot transcription as a message.""" if len(self._bot_transcription) > 0: message = RTVIBotTranscriptionMessage( data=RTVITextMessageData(text=self._bot_transcription) @@ -531,6 +776,7 @@ class RTVIObserver(BaseObserver): self._bot_transcription = "" async def _handle_interruptions(self, frame: Frame): + """Handle user speaking interruption frames.""" message = None if isinstance(frame, UserStartedSpeakingFrame): message = RTVIUserStartedSpeakingMessage() @@ -541,6 +787,7 @@ class RTVIObserver(BaseObserver): await self.push_transport_message_urgent(message) async def _handle_bot_speaking(self, frame: Frame): + """Handle bot speaking event frames.""" message = None if isinstance(frame, BotStartedSpeakingFrame): message = RTVIBotStartedSpeakingMessage() @@ -551,6 +798,7 @@ class RTVIObserver(BaseObserver): await self.push_transport_message_urgent(message) async def _handle_llm_text_frame(self, frame: LLMTextFrame): + """Handle LLM text output frames.""" message = RTVIBotLLMTextMessage(data=RTVITextMessageData(text=frame.text)) await self.push_transport_message_urgent(message) @@ -559,6 +807,7 @@ class RTVIObserver(BaseObserver): await self._push_bot_transcription() async def _handle_user_transcriptions(self, frame: Frame): + """Handle user transcription frames.""" message = None if isinstance(frame, TranscriptionFrame): message = RTVIUserTranscriptionMessage( @@ -607,6 +856,7 @@ class RTVIObserver(BaseObserver): logger.warning(f"Caught an error while trying to handle context: {e}") async def _handle_metrics(self, frame: MetricsFrame): + """Handle metrics frames and convert to RTVI metrics messages.""" metrics = {} for d in frame.data: if isinstance(d, TTFBMetricsData): @@ -631,6 +881,13 @@ class RTVIObserver(BaseObserver): class RTVIProcessor(FrameProcessor): + """Main processor for handling RTVI protocol messages and actions. + + This processor manages the RTVI protocol communication including client-server + handshaking, configuration management, action execution, and message routing. + It serves as the central hub for RTVI protocol operations. + """ + def __init__( self, *, @@ -638,6 +895,13 @@ class RTVIProcessor(FrameProcessor): transport: Optional[BaseTransport] = None, **kwargs, ): + """Initialize the RTVI processor. + + Args: + config: Initial RTVI configuration. + transport: Transport layer for communication. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._config = config or RTVIConfig(config=[]) @@ -650,11 +914,9 @@ class RTVIProcessor(FrameProcessor): self._registered_services: Dict[str, RTVIService] = {} # A task to process incoming action frames. - self._action_queue = asyncio.Queue() self._action_task: Optional[asyncio.Task] = None # A task to process incoming transport messages. - self._message_queue = asyncio.Queue() self._message_task: Optional[asyncio.Task] = None self._register_event_handler("on_bot_started") @@ -669,34 +931,67 @@ class RTVIProcessor(FrameProcessor): self._input_transport.enable_audio_in_stream_on_start(False) def register_action(self, action: RTVIAction): + """Register an action that can be executed via RTVI. + + Args: + action: The action to register. + """ id = self._action_id(action.service, action.action) self._registered_actions[id] = action def register_service(self, service: RTVIService): + """Register a service that can be configured via RTVI. + + Args: + service: The service to register. + """ self._registered_services[service.name] = service async def set_client_ready(self): + """Mark the client as ready and trigger the ready event.""" self._client_ready = True await self._call_event_handler("on_client_ready") async def set_bot_ready(self): + """Mark the bot as ready and send the bot-ready message.""" self._bot_ready = True await self._update_config(self._config, False) await self._send_bot_ready() def set_errors_enabled(self, enabled: bool): + """Enable or disable error message sending. + + Args: + enabled: Whether to send error messages. + """ self._errors_enabled = enabled async def interrupt_bot(self): + """Send a bot interruption frame upstream.""" await self.push_frame(BotInterruptionFrame(), FrameDirection.UPSTREAM) async def send_error(self, error: str): + """Send an error message to the client. + + Args: + error: The error message to send. + """ await self._send_error_frame(ErrorFrame(error=error)) async def handle_message(self, message: RTVIMessage): + """Handle an incoming RTVI message. + + Args: + message: The RTVI message to handle. + """ await self._message_queue.put(message) async def handle_function_call(self, params: FunctionCallParams): + """Handle a function call from the LLM. + + Args: + params: The function call parameters. + """ fn = RTVILLMFunctionCallMessageData( function_name=params.function_name, tool_call_id=params.tool_call_id, @@ -708,6 +1003,16 @@ class RTVIProcessor(FrameProcessor): async def handle_function_call_start( self, function_name: str, llm: FrameProcessor, context: OpenAILLMContext ): + """Handle the start of a function call from the LLM. + + Args: + function_name: Name of the function being called. + llm: The LLM processor making the call. + context: The LLM context. + + Note: + This method is deprecated. Use handle_function_call() instead. + """ import warnings with warnings.catch_warnings(): @@ -722,6 +1027,12 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message, exclude_none=False) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames through the RTVI processor. + + Args: + frame: The frame to process. + direction: The direction of frame flow. + """ await super().process_frame(frame, direction) # Specific system frames @@ -755,19 +1066,25 @@ class RTVIProcessor(FrameProcessor): await self.push_frame(frame, direction) async def _start(self, frame: StartFrame): + """Start the RTVI processor tasks.""" if not self._action_task: + self._action_queue = WatchdogQueue(self.task_manager) self._action_task = self.create_task(self._action_task_handler()) if not self._message_task: + self._message_queue = WatchdogQueue(self.task_manager) self._message_task = self.create_task(self._message_task_handler()) await self._call_event_handler("on_bot_started") async def _stop(self, frame: EndFrame): + """Stop the RTVI processor tasks.""" await self._cancel_tasks() async def _cancel(self, frame: CancelFrame): + """Cancel the RTVI processor tasks.""" await self._cancel_tasks() async def _cancel_tasks(self): + """Cancel all running tasks.""" if self._action_task: await self.cancel_task(self._action_task) self._action_task = None @@ -777,26 +1094,26 @@ class RTVIProcessor(FrameProcessor): self._message_task = None async def _push_transport_message(self, model: BaseModel, exclude_none: bool = True): + """Push a transport message frame.""" frame = TransportMessageUrgentFrame(message=model.model_dump(exclude_none=exclude_none)) await self.push_frame(frame) async def _action_task_handler(self): + """Handle incoming action frames.""" while True: frame = await self._action_queue.get() - self.start_watchdog() await self._handle_action(frame.message_id, frame.rtvi_action_run) self._action_queue.task_done() - self.reset_watchdog() async def _message_task_handler(self): + """Handle incoming transport messages.""" while True: message = await self._message_queue.get() - self.start_watchdog() await self._handle_message(message) self._message_queue.task_done() - self.reset_watchdog() async def _handle_transport_message(self, frame: TransportMessageUrgentFrame): + """Handle an incoming transport message frame.""" try: transport_message = frame.message if transport_message.get("label") != RTVI_MESSAGE_LABEL: @@ -809,6 +1126,7 @@ class RTVIProcessor(FrameProcessor): logger.warning(f"Invalid RTVI transport message: {e}") async def _handle_message(self, message: RTVIMessage): + """Handle a parsed RTVI message.""" try: match message.type: case "client-ready": @@ -845,6 +1163,7 @@ class RTVIProcessor(FrameProcessor): logger.warning(f"Exception processing message: {e}") async def _handle_client_ready(self, request_id: str): + """Handle a client-ready message.""" logger.debug("Received client-ready") if self._input_transport: await self._input_transport.start_audio_in_streaming() @@ -853,6 +1172,7 @@ class RTVIProcessor(FrameProcessor): await self.set_client_ready() async def _handle_audio_buffer(self, data): + """Handle incoming audio buffer data.""" if not self._input_transport: return @@ -874,20 +1194,24 @@ class RTVIProcessor(FrameProcessor): logger.error(f"Error processing audio buffer: {e}") async def _handle_describe_config(self, request_id: str): + """Handle a describe-config request.""" services = list(self._registered_services.values()) message = RTVIDescribeConfig(id=request_id, data=RTVIDescribeConfigData(config=services)) await self._push_transport_message(message) async def _handle_describe_actions(self, request_id: str): + """Handle a describe-actions request.""" actions = list(self._registered_actions.values()) message = RTVIDescribeActions(id=request_id, data=RTVIDescribeActionsData(actions=actions)) await self._push_transport_message(message) async def _handle_get_config(self, request_id: str): + """Handle a get-config request.""" message = RTVIConfigResponse(id=request_id, data=self._config) await self._push_transport_message(message) def _update_config_option(self, service: str, config: RTVIServiceOptionConfig): + """Update a specific configuration option.""" for service_config in self._config.config: if service_config.service == service: for option_config in service_config.options: @@ -899,6 +1223,7 @@ class RTVIProcessor(FrameProcessor): service_config.options.append(config) async def _update_service_config(self, config: RTVIServiceConfig): + """Update configuration for a specific service.""" service = self._registered_services[config.service] for option in config.options: handler = service._options_dict[option.name].handler @@ -906,16 +1231,19 @@ class RTVIProcessor(FrameProcessor): self._update_config_option(service.name, option) async def _update_config(self, data: RTVIConfig, interrupt: bool): + """Update the RTVI configuration.""" if interrupt: await self.interrupt_bot() for service_config in data.config: await self._update_service_config(service_config) async def _handle_update_config(self, request_id: str, data: RTVIUpdateConfig): + """Handle an update-config request.""" await self._update_config(RTVIConfig(config=data.config), data.interrupt) await self._handle_get_config(request_id) async def _handle_function_call_result(self, data): + """Handle a function call result from the client.""" frame = FunctionCallResultFrame( function_name=data.function_name, tool_call_id=data.tool_call_id, @@ -925,6 +1253,7 @@ class RTVIProcessor(FrameProcessor): await self.push_frame(frame) async def _handle_action(self, request_id: Optional[str], data: RTVIActionRun): + """Handle an action execution request.""" action_id = self._action_id(data.service, data.action) if action_id not in self._registered_actions: await self._send_error_response(request_id, f"Action {action_id} not registered") @@ -942,6 +1271,7 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message) async def _send_bot_ready(self): + """Send the bot-ready message to the client.""" message = RTVIBotReady( id=self._client_ready_id, data=RTVIBotReadyData(version=RTVI_PROTOCOL_VERSION, config=self._config.config), @@ -949,14 +1279,17 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message) async def _send_error_frame(self, frame: ErrorFrame): + """Send an error frame as an RTVI error message.""" if self._errors_enabled: message = RTVIError(data=RTVIErrorData(error=frame.error, fatal=frame.fatal)) await self._push_transport_message(message) async def _send_error_response(self, id: str, error: str): + """Send an error response message.""" if self._errors_enabled: message = RTVIErrorResponse(id=id, data=RTVIErrorResponseData(error=error)) await self._push_transport_message(message) def _action_id(self, service: str, action: str) -> str: + """Generate an action ID from service and action names.""" return f"{service}:{action}" diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 84b8a050b..15f483029 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""GStreamer pipeline source integration for Pipecat.""" + import asyncio from typing import Optional @@ -36,7 +38,24 @@ except ModuleNotFoundError as e: class GStreamerPipelineSource(FrameProcessor): + """A frame processor that uses GStreamer pipelines as media sources. + + This processor creates and manages GStreamer pipelines to generate audio and video + output frames. It handles pipeline lifecycle, decoding, format conversion, and + frame generation with configurable output parameters. + """ + class OutputParams(BaseModel): + """Output configuration parameters for GStreamer pipeline. + + Parameters: + video_width: Width of output video frames in pixels. + video_height: Height of output video frames in pixels. + audio_sample_rate: Sample rate for audio output. If None, uses frame sample rate. + audio_channels: Number of audio channels for output. + clock_sync: Whether to synchronize output with pipeline clock. + """ + video_width: int = 1280 video_height: int = 720 audio_sample_rate: Optional[int] = None @@ -44,6 +63,13 @@ class GStreamerPipelineSource(FrameProcessor): clock_sync: bool = True def __init__(self, *, pipeline: str, out_params: Optional[OutputParams] = None, **kwargs): + """Initialize the GStreamer pipeline source. + + Args: + pipeline: GStreamer pipeline description string for the source. + out_params: Output configuration parameters. If None, uses defaults. + **kwargs: Additional arguments passed to parent FrameProcessor. + """ super().__init__(**kwargs) self._out_params = out_params or GStreamerPipelineSource.OutputParams() @@ -67,6 +93,12 @@ class GStreamerPipelineSource(FrameProcessor): bus.connect("message", self._on_gstreamer_message) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and manage GStreamer pipeline lifecycle. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) # Specific system frames @@ -92,13 +124,16 @@ class GStreamerPipelineSource(FrameProcessor): await self.push_frame(frame, direction) async def _start(self, frame: StartFrame): + """Start the GStreamer pipeline.""" self._sample_rate = self._out_params.audio_sample_rate or frame.audio_out_sample_rate self._player.set_state(Gst.State.PLAYING) async def _stop(self, frame: EndFrame): + """Stop the GStreamer pipeline.""" self._player.set_state(Gst.State.NULL) async def _cancel(self, frame: CancelFrame): + """Cancel the GStreamer pipeline.""" self._player.set_state(Gst.State.NULL) # @@ -106,6 +141,7 @@ class GStreamerPipelineSource(FrameProcessor): # def _on_gstreamer_message(self, bus: Gst.Bus, message: Gst.Message): + """Handle GStreamer bus messages.""" t = message.type if t == Gst.MessageType.ERROR: err, debug = message.parse_error() @@ -113,6 +149,7 @@ class GStreamerPipelineSource(FrameProcessor): return True def _decodebin_callback(self, decodebin: Gst.Element, pad: Gst.Pad): + """Handle new pads from decodebin element.""" caps_string = pad.get_current_caps().to_string() if caps_string.startswith("audio"): self._decodebin_audio(pad) @@ -120,6 +157,7 @@ class GStreamerPipelineSource(FrameProcessor): self._decodebin_video(pad) def _decodebin_audio(self, pad: Gst.Pad): + """Set up audio processing pipeline from decoded audio pad.""" queue_audio = Gst.ElementFactory.make("queue", None) audioconvert = Gst.ElementFactory.make("audioconvert", None) audioresample = Gst.ElementFactory.make("audioresample", None) @@ -153,6 +191,7 @@ class GStreamerPipelineSource(FrameProcessor): pad.link(queue_pad) def _decodebin_video(self, pad: Gst.Pad): + """Set up video processing pipeline from decoded video pad.""" queue_video = Gst.ElementFactory.make("queue", None) videoconvert = Gst.ElementFactory.make("videoconvert", None) videoscale = Gst.ElementFactory.make("videoscale", None) @@ -187,6 +226,7 @@ class GStreamerPipelineSource(FrameProcessor): pad.link(queue_pad) def _appsink_audio_new_sample(self, appsink: GstApp.AppSink): + """Handle new audio samples from GStreamer appsink.""" buffer = appsink.pull_sample().get_buffer() (_, info) = buffer.map(Gst.MapFlags.READ) frame = OutputAudioRawFrame( @@ -199,6 +239,7 @@ class GStreamerPipelineSource(FrameProcessor): return Gst.FlowReturn.OK def _appsink_video_new_sample(self, appsink: GstApp.AppSink): + """Handle new video samples from GStreamer appsink.""" buffer = appsink.pull_sample().get_buffer() (_, info) = buffer.map(Gst.MapFlags.READ) frame = OutputImageRawFrame( diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index 36c7e9821..672027491 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Idle frame processor for timeout-based callback execution.""" + import asyncio from typing import Awaitable, Callable, List, Optional @@ -12,9 +14,11 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class IdleFrameProcessor(FrameProcessor): - """This class waits to receive any frame or list of desired frames within a - given timeout. If the timeout is reached before receiving any of those - frames the provided callback will be called. + """Monitors frame activity and triggers callbacks on timeout. + + This processor waits to receive any frame or specific frame types within a + given timeout period. If the timeout is reached before receiving the expected + frames, the provided callback will be executed. """ def __init__( @@ -25,6 +29,16 @@ class IdleFrameProcessor(FrameProcessor): types: Optional[List[type]] = None, **kwargs, ): + """Initialize the idle frame processor. + + Args: + callback: Async callback function to execute on timeout. Receives + this processor instance as an argument. + timeout: Timeout duration in seconds before triggering the callback. + types: Optional list of frame types to monitor. If None, monitors + all frames. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._callback = callback @@ -33,6 +47,12 @@ class IdleFrameProcessor(FrameProcessor): self._idle_task = None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and manage idle timeout monitoring. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -50,15 +70,18 @@ class IdleFrameProcessor(FrameProcessor): self._idle_event.set() async def cleanup(self): + """Clean up resources and cancel pending tasks.""" if self._idle_task: await self.cancel_task(self._idle_task) def _create_idle_task(self): + """Create and start the idle monitoring task.""" if not self._idle_task: self._idle_event = asyncio.Event() self._idle_task = self.create_task(self._idle_task_handler()) async def _idle_task_handler(self): + """Handle idle timeout monitoring and callback execution.""" while True: try: await asyncio.wait_for(self._idle_event.wait(), timeout=self._timeout) diff --git a/src/pipecat/processors/logger.py b/src/pipecat/processors/logger.py index b773df8c3..acc85c40b 100644 --- a/src/pipecat/processors/logger.py +++ b/src/pipecat/processors/logger.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame logging utilities for debugging and monitoring frame flow in Pipecat pipelines.""" + from typing import Optional, Tuple, Type from loguru import logger @@ -21,6 +23,13 @@ logger = logger.opt(ansi=True) class FrameLogger(FrameProcessor): + """A frame processor that logs frame information for debugging purposes. + + This processor intercepts frames passing through the pipeline and logs + their details with configurable formatting and filtering. Useful for + debugging frame flow and understanding pipeline behavior. + """ + def __init__( self, prefix="Frame", @@ -32,12 +41,26 @@ class FrameLogger(FrameProcessor): TransportMessageFrame, ), ): + """Initialize the frame logger. + + Args: + prefix: Text prefix to add to log messages. Defaults to "Frame". + color: ANSI color code for log message formatting. If None, no coloring is applied. + ignored_frame_types: Tuple of frame types to exclude from logging. + Defaults to common high-frequency frames like audio and speaking frames. + """ super().__init__() self._prefix = prefix self._color = color self._ignored_frame_types = ignored_frame_types async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process and log frame information. + + Args: + frame: The frame to process and potentially log. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if self._ignored_frame_types and not isinstance(frame, self._ignored_frame_types): diff --git a/src/pipecat/processors/metrics/frame_processor_metrics.py b/src/pipecat/processors/metrics/frame_processor_metrics.py index 40a83fa38..fd93241ed 100644 --- a/src/pipecat/processors/metrics/frame_processor_metrics.py +++ b/src/pipecat/processors/metrics/frame_processor_metrics.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Frame processor metrics collection and reporting.""" + import time from typing import Optional @@ -18,12 +20,25 @@ from pipecat.metrics.metrics import ( TTFBMetricsData, TTSUsageMetricsData, ) -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio.task_manager import BaseTaskManager from pipecat.utils.base_object import BaseObject class FrameProcessorMetrics(BaseObject): + """Metrics collection and reporting for frame processors. + + Provides comprehensive metrics tracking for frame processing operations, + including timing measurements, resource usage, and performance analytics. + Supports TTFB tracking, processing duration metrics, and usage statistics + for LLM and TTS operations. + """ + def __init__(self): + """Initialize the frame processor metrics collector. + + Sets up internal state for tracking various metrics including TTFB, + processing times, and usage statistics. + """ super().__init__() self._task_manager = None self._start_ttfb_time = 0 @@ -31,14 +46,25 @@ class FrameProcessorMetrics(BaseObject): self._last_ttfb_time = 0 self._should_report_ttfb = True - async def setup(self, task_manager: TaskManager): + async def setup(self, task_manager: BaseTaskManager): + """Set up the metrics collector with a task manager. + + Args: + task_manager: The task manager for handling async operations. + """ self._task_manager = task_manager async def cleanup(self): + """Clean up metrics collection resources.""" await super().cleanup() @property - def task_manager(self) -> TaskManager: + def task_manager(self) -> BaseTaskManager: + """Get the associated task manager. + + Returns: + The task manager instance for async operations. + """ return self._task_manager @property @@ -46,7 +72,7 @@ class FrameProcessorMetrics(BaseObject): """Get the current TTFB value in seconds. Returns: - Optional[float]: The TTFB value in seconds, or None if not measured + The TTFB value in seconds, or None if not measured. """ if self._last_ttfb_time > 0: return self._last_ttfb_time @@ -58,24 +84,46 @@ class FrameProcessorMetrics(BaseObject): return None def _processor_name(self): + """Get the processor name from core metrics data.""" return self._core_metrics_data.processor def _model_name(self): + """Get the model name from core metrics data.""" return self._core_metrics_data.model def set_core_metrics_data(self, data: MetricsData): + """Set the core metrics data for this collector. + + Args: + data: The core metrics data containing processor and model information. + """ self._core_metrics_data = data def set_processor_name(self, name: str): + """Set the processor name for metrics reporting. + + Args: + name: The name of the processor to use in metrics. + """ self._core_metrics_data = MetricsData(processor=name) async def start_ttfb_metrics(self, report_only_initial_ttfb): + """Start measuring time-to-first-byte (TTFB). + + Args: + report_only_initial_ttfb: Whether to report only the first TTFB measurement. + """ if self._should_report_ttfb: self._start_ttfb_time = time.time() self._last_ttfb_time = 0 self._should_report_ttfb = not report_only_initial_ttfb async def stop_ttfb_metrics(self): + """Stop TTFB measurement and generate metrics frame. + + Returns: + MetricsFrame containing TTFB data, or None if not measuring. + """ if self._start_ttfb_time == 0: return None @@ -88,9 +136,15 @@ class FrameProcessorMetrics(BaseObject): return MetricsFrame(data=[ttfb]) async def start_processing_metrics(self): + """Start measuring processing time.""" self._start_processing_time = time.time() async def stop_processing_metrics(self): + """Stop processing time measurement and generate metrics frame. + + Returns: + MetricsFrame containing processing duration data, or None if not measuring. + """ if self._start_processing_time == 0: return None @@ -103,15 +157,34 @@ class FrameProcessorMetrics(BaseObject): return MetricsFrame(data=[processing]) async def start_llm_usage_metrics(self, tokens: LLMTokenUsage): - logger.debug( - f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}" - ) + """Record LLM token usage metrics. + + Args: + tokens: Token usage information including prompt and completion tokens. + + Returns: + MetricsFrame containing LLM usage data. + """ + logstr = f"{self._processor_name()} prompt tokens: {tokens.prompt_tokens}, completion tokens: {tokens.completion_tokens}" + if tokens.cache_read_input_tokens: + logstr += f", cache read input tokens: {tokens.cache_read_input_tokens}" + if tokens.reasoning_tokens: + logstr += f", reasoning tokens: {tokens.reasoning_tokens}" + logger.debug(logstr) value = LLMUsageMetricsData( processor=self._processor_name(), model=self._model_name(), value=tokens ) return MetricsFrame(data=[value]) async def start_tts_usage_metrics(self, text: str): + """Record TTS character usage metrics. + + Args: + text: The text being processed by TTS. + + Returns: + MetricsFrame containing TTS usage data. + """ characters = TTSUsageMetricsData( processor=self._processor_name(), model=self._model_name(), value=len(text) ) diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index 20d854ffb..755d5cd5e 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -4,11 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio +"""Sentry integration for frame processor metrics.""" from loguru import logger -from pipecat.utils.asyncio import TaskManager +from pipecat.utils.asyncio.task_manager import BaseTaskManager +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue try: import sentry_sdk @@ -21,25 +22,45 @@ from pipecat.processors.metrics.frame_processor_metrics import FrameProcessorMet class SentryMetrics(FrameProcessorMetrics): + """Frame processor metrics integration with Sentry monitoring. + + Extends FrameProcessorMetrics to send time-to-first-byte (TTFB) and + processing metrics as Sentry transactions for performance monitoring + and debugging. + """ + def __init__(self): + """Initialize the Sentry metrics collector. + + Sets up internal state for tracking transactions and verifies + Sentry SDK initialization status. + """ super().__init__() self._ttfb_metrics_tx = None self._processing_metrics_tx = None self._sentry_available = sentry_sdk.is_initialized() if not self._sentry_available: logger.warning("Sentry SDK not initialized. Sentry features will be disabled.") - self._sentry_queue = asyncio.Queue() self._sentry_task = None - async def setup(self, task_manager: TaskManager): + async def setup(self, task_manager: BaseTaskManager): + """Setup the Sentry metrics system. + + Args: + task_manager: The task manager to use for background operations. + """ await super().setup(task_manager) if self._sentry_available: - self._sentry_queue = asyncio.Queue() + self._sentry_queue = WatchdogQueue(task_manager) self._sentry_task = self.task_manager.create_task( self._sentry_task_handler(), name=f"{self}::_sentry_task_handler" ) async def cleanup(self): + """Clean up Sentry resources and flush pending transactions. + + Ensures all pending transactions are sent to Sentry before shutdown. + """ await super().cleanup() if self._sentry_task: await self._sentry_queue.put(None) @@ -49,6 +70,11 @@ class SentryMetrics(FrameProcessorMetrics): sentry_sdk.flush(timeout=5.0) async def start_ttfb_metrics(self, report_only_initial_ttfb): + """Start tracking time-to-first-byte metrics. + + Args: + report_only_initial_ttfb: Whether to report only the initial TTFB measurement. + """ await super().start_ttfb_metrics(report_only_initial_ttfb) if self._should_report_ttfb and self._sentry_available: @@ -61,6 +87,10 @@ class SentryMetrics(FrameProcessorMetrics): ) async def stop_ttfb_metrics(self): + """Stop tracking time-to-first-byte metrics. + + Queues the TTFB transaction for completion and transmission to Sentry. + """ await super().stop_ttfb_metrics() if self._sentry_available and self._ttfb_metrics_tx: @@ -68,6 +98,10 @@ class SentryMetrics(FrameProcessorMetrics): self._ttfb_metrics_tx = None async def start_processing_metrics(self): + """Start tracking frame processing metrics. + + Creates a new Sentry transaction to track processing performance. + """ await super().start_processing_metrics() if self._sentry_available: @@ -80,6 +114,10 @@ class SentryMetrics(FrameProcessorMetrics): ) async def stop_processing_metrics(self): + """Stop tracking frame processing metrics. + + Queues the processing transaction for completion and transmission to Sentry. + """ await super().stop_processing_metrics() if self._sentry_available and self._processing_metrics_tx: @@ -87,9 +125,11 @@ class SentryMetrics(FrameProcessorMetrics): self._processing_metrics_tx = None async def _sentry_task_handler(self): + """Background task handler for completing Sentry transactions.""" running = True while running: tx = await self._sentry_queue.get() if tx: await self.task_manager.get_event_loop().run_in_executor(None, tx.finish) running = tx is not None + self._sentry_queue.task_done() diff --git a/src/pipecat/processors/producer_processor.py b/src/pipecat/processors/producer_processor.py index 6ada2ed83..8de9d66bb 100644 --- a/src/pipecat/processors/producer_processor.py +++ b/src/pipecat/processors/producer_processor.py @@ -4,23 +4,35 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Producer processor for frame filtering and distribution.""" + import asyncio from typing import Awaitable, Callable, List from pipecat.frames.frames import Frame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue async def identity_transformer(frame: Frame): + """Default transformer that returns the frame unchanged. + + Args: + frame: The frame to transform. + + Returns: + The same frame without modifications. + """ return frame class ProducerProcessor(FrameProcessor): - """This class optionally passes-through received frames and decides if those - frames should be sent to consumers based on a user-defined filter. The - frames can be transformed into a different type of frame before being - sending them to the consumers. More than one consumer can be added. + """A processor that filters frames and distributes them to multiple consumers. + This processor receives frames, applies a filter to determine which frames + should be sent to consumers (ConsumerProcessor), optionally transforms those + frames, and distributes them to registered consumer queues. It can also pass + frames through to the next processor in the pipeline. """ def __init__( @@ -30,6 +42,16 @@ class ProducerProcessor(FrameProcessor): transformer: Callable[[Frame], Awaitable[Frame]] = identity_transformer, passthrough: bool = True, ): + """Initialize the producer processor. + + Args: + filter: Async function that determines if a frame should be produced. + Must return True for frames to be sent to consumers. + transformer: Async function to transform frames before sending to consumers. + Defaults to identity_transformer which returns frames unchanged. + passthrough: Whether to pass frames through to the next processor. + If True, all frames continue downstream regardless of filter result. + """ super().__init__() self._filter = filter self._transformer = transformer @@ -37,26 +59,25 @@ class ProducerProcessor(FrameProcessor): self._consumers: List[asyncio.Queue] = [] def add_consumer(self): - """ - Adds a new consumer and returns its associated queue. + """Add a new consumer and return its associated queue. Returns: asyncio.Queue: The queue for the newly added consumer. """ - queue = asyncio.Queue() + queue = WatchdogQueue(self.task_manager) self._consumers.append(queue) return queue async def process_frame(self, frame: Frame, direction: FrameDirection): - """ - Processes an incoming frame and determines whether to produce it as a ProducerItem. + """Process an incoming frame and determine whether to produce it. - If the frame meets the produce criteria, it will be added to the consumer queues. - If passthrough is enabled, the frame will also be sent to consumers. + If the frame meets the filter criteria, it will be transformed and added + to all consumer queues. If passthrough is enabled, the original frame + will also be sent downstream. Args: - frame (Frame): The frame to process. - direction (FrameDirection): The direction of the frame. + frame: The frame to process. + direction: The direction of the frame flow. """ await super().process_frame(frame, direction) @@ -68,6 +89,7 @@ class ProducerProcessor(FrameProcessor): await self.push_frame(frame, direction) async def _produce(self, frame: Frame): + """Produce a frame to all consumers.""" for consumer in self._consumers: new_frame = await self._transformer(frame) await consumer.put(new_frame) diff --git a/src/pipecat/processors/text_transformer.py b/src/pipecat/processors/text_transformer.py index 9b563e187..7dcef31df 100644 --- a/src/pipecat/processors/text_transformer.py +++ b/src/pipecat/processors/text_transformer.py @@ -4,14 +4,20 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import Coroutine +"""Stateless text transformation processor for Pipecat.""" + +from typing import Callable, Coroutine, Union from pipecat.frames.frames import Frame, TextFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class StatelessTextTransformer(FrameProcessor): - """This processor calls the given function on any text in a text frame. + """Processor that applies transformation functions to text frames. + + This processor intercepts TextFrame objects and applies a user-provided + transformation function to the text content. The function can be either + synchronous or asynchronous (coroutine). >>> async def print_frames(aggregator, frame): ... async for frame in aggregator.process_frame(frame): @@ -22,11 +28,25 @@ class StatelessTextTransformer(FrameProcessor): HELLO """ - def __init__(self, transform_fn): + def __init__( + self, transform_fn: Union[Callable[[str], str], Callable[[str], Coroutine[None, None, str]]] + ): + """Initialize the text transformer. + + Args: + transform_fn: Function to apply to text content. Can be synchronous + (str -> str) or asynchronous (str -> Coroutine[None, None, str]). + """ super().__init__() self._transform_fn = transform_fn async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames, applying transformation to text frames. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, TextFrame): diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 97ccd0cd4..856311392 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Transcript processing utilities for conversation recording and analysis. + +This module provides processors that convert speech and text frames into structured +transcript messages with timestamps, enabling conversation history tracking and analysis. +""" + from typing import List, Optional from loguru import logger @@ -30,7 +36,11 @@ class BaseTranscriptProcessor(FrameProcessor): """ def __init__(self, **kwargs): - """Initialize processor with empty message store.""" + """Initialize processor with empty message store. + + Args: + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._processed_messages: List[TranscriptionMessage] = [] self._register_event_handler("on_transcript_update") @@ -39,7 +49,7 @@ class BaseTranscriptProcessor(FrameProcessor): """Emit transcript updates for new messages. Args: - messages: New messages to emit in update + messages: New messages to emit in update. """ if messages: self._processed_messages.extend(messages) @@ -55,8 +65,8 @@ class UserTranscriptProcessor(BaseTranscriptProcessor): """Process TranscriptionFrames into user conversation messages. Args: - frame: Input frame to process - direction: Frame processing direction + frame: Input frame to process. + direction: Frame processing direction. """ await super().process_frame(frame, direction) @@ -77,14 +87,14 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): - The bot stops speaking (BotStoppedSpeakingFrame) - The bot is interrupted (StartInterruptionFrame) - The pipeline ends (EndFrame) - - Attributes: - _current_text_parts: List of text fragments being aggregated for current utterance - _aggregation_start_time: Timestamp when the current utterance began """ def __init__(self, **kwargs): - """Initialize processor with aggregation state.""" + """Initialize processor with aggregation state. + + Args: + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) self._current_text_parts: List[str] = [] self._aggregation_start_time: Optional[str] = None @@ -176,8 +186,8 @@ class AssistantTranscriptProcessor(BaseTranscriptProcessor): - CancelFrame: Completes current utterance due to cancellation Args: - frame: Input frame to process - direction: Frame processing direction + frame: Input frame to process. + direction: Frame processing direction. """ await super().process_frame(frame, direction) @@ -245,7 +255,10 @@ class TranscriptProcessor: """Get the user transcript processor. Args: - **kwargs: Arguments specific to UserTranscriptProcessor + **kwargs: Arguments specific to UserTranscriptProcessor. + + Returns: + The user transcript processor instance. """ if self._user_processor is None: self._user_processor = UserTranscriptProcessor(**kwargs) @@ -262,7 +275,10 @@ class TranscriptProcessor: """Get the assistant transcript processor. Args: - **kwargs: Arguments specific to AssistantTranscriptProcessor + **kwargs: Arguments specific to AssistantTranscriptProcessor. + + Returns: + The assistant transcript processor instance. """ if self._assistant_processor is None: self._assistant_processor = AssistantTranscriptProcessor(**kwargs) @@ -279,10 +295,10 @@ class TranscriptProcessor: """Register event handler for both processors. Args: - event_name: Name of event to handle + event_name: Name of event to handle. Returns: - Decorator function that registers handler with both processors + Decorator function that registers handler with both processors. """ def decorator(handler): diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index e7b08f4a3..c692642cc 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""User idle detection and timeout handling for Pipecat.""" + import asyncio import inspect from typing import Awaitable, Callable, Union @@ -22,19 +24,12 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class UserIdleProcessor(FrameProcessor): """Monitors user inactivity and triggers callbacks after timeout periods. - Starts monitoring only after the first conversation activity (UserStartedSpeaking - or BotSpeaking). - - Args: - callback: Function to call when user is idle. Can be either: - - Basic callback(processor) -> None - - Retry callback(processor, retry_count) -> bool - Return True to continue monitoring for idle events, - Return False to stop the idle monitoring task - timeout: Seconds to wait before considering user idle - **kwargs: Additional arguments passed to FrameProcessor + This processor tracks user activity and triggers configurable callbacks when + users become idle. It starts monitoring only after the first conversation + activity and supports both basic and retry-based callback patterns. Example: + ``` # Retry callback: async def handle_idle(processor: "UserIdleProcessor", retry_count: int) -> bool: if retry_count < 3: @@ -50,6 +45,7 @@ class UserIdleProcessor(FrameProcessor): callback=handle_idle, timeout=5.0 ) + ``` """ def __init__( @@ -62,6 +58,17 @@ class UserIdleProcessor(FrameProcessor): timeout: float, **kwargs, ): + """Initialize the user idle processor. + + Args: + callback: Function to call when user is idle. Can be either: + - Basic callback(processor) -> None + - Retry callback(processor, retry_count) -> bool + Return True to continue monitoring for idle events, + Return False to stop the idle monitoring task + timeout: Seconds to wait before considering user idle. + **kwargs: Additional arguments passed to FrameProcessor. + """ super().__init__(**kwargs) self._callback = self._wrap_callback(callback) self._timeout = timeout @@ -107,7 +114,11 @@ class UserIdleProcessor(FrameProcessor): @property def retry_count(self) -> int: - """Returns the current retry count.""" + """Get the current retry count. + + Returns: + The number of times the idle callback has been triggered. + """ return self._retry_count async def _stop(self) -> None: @@ -120,8 +131,8 @@ class UserIdleProcessor(FrameProcessor): """Processes incoming frames and manages idle monitoring state. Args: - frame: The frame to process - direction: Direction of the frame flow + frame: The frame to process. + direction: Direction of the frame flow. """ await super().process_frame(frame, direction) diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index 985b61c8e..eebdbfb0b 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base AI service implementation. + +Provides the foundation for all AI services in the Pipecat framework, including +model management, settings handling, and frame processing lifecycle methods. +""" + from typing import Any, AsyncGenerator, Dict, Mapping from loguru import logger @@ -20,7 +26,20 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class AIService(FrameProcessor): + """Base class for all AI services. + + Provides common functionality for AI services including model management, + settings handling, session properties, and frame processing lifecycle. + Subclasses should implement specific AI functionality while leveraging + this base infrastructure. + """ + def __init__(self, **kwargs): + """Initialize the AI service. + + Args: + **kwargs: Additional arguments passed to the parent FrameProcessor. + """ super().__init__(**kwargs) self._model_name: str = "" self._settings: Dict[str, Any] = {} @@ -28,19 +47,53 @@ class AIService(FrameProcessor): @property def model_name(self) -> str: + """Get the current model name. + + Returns: + The name of the AI model being used. + """ return self._model_name def set_model_name(self, model: str): + """Set the AI model name and update metrics. + + Args: + model: The name of the AI model to use. + """ self._model_name = model self.set_core_metrics_data(MetricsData(processor=self.name, model=self._model_name)) async def start(self, frame: StartFrame): + """Start the AI service. + + Called when the service should begin processing. Subclasses should + override this method to perform service-specific initialization. + + Args: + frame: The start frame containing initialization parameters. + """ pass async def stop(self, frame: EndFrame): + """Stop the AI service. + + Called when the service should stop processing. Subclasses should + override this method to perform cleanup operations. + + Args: + frame: The end frame. + """ pass async def cancel(self, frame: CancelFrame): + """Cancel the AI service. + + Called when the service should cancel all operations. Subclasses should + override this method to handle cancellation logic. + + Args: + frame: The cancel frame. + """ pass async def _update_settings(self, settings: Mapping[str, Any]): @@ -87,6 +140,15 @@ class AIService(FrameProcessor): logger.warning(f"Unknown setting for {self.name} service: {key}") async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames and handle service lifecycle. + + Automatically handles StartFrame, EndFrame, and CancelFrame by calling + the appropriate lifecycle methods. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): @@ -97,6 +159,14 @@ class AIService(FrameProcessor): await self.stop(frame) async def process_generator(self, generator: AsyncGenerator[Frame | None, None]): + """Process frames from an async generator. + + Takes an async generator that yields frames and processes each one, + handling error frames specially by pushing them as errors. + + Args: + generator: An async generator that yields Frame objects or None. + """ async for f in generator: if f: if isinstance(f, ErrorFrame): diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index 1a1e1ab56..833a20eae 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -4,6 +4,17 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Deprecated AI services module. + +This module is deprecated. Import services directly from their respective modules: +- pipecat.services.ai_service +- pipecat.services.image_service +- pipecat.services.llm_service +- pipecat.services.stt_service +- pipecat.services.tts_service +- pipecat.services.vision_service +""" + import sys from pipecat.services import DeprecatedModuleProxy diff --git a/src/pipecat/services/anthropic/llm.py b/src/pipecat/services/anthropic/llm.py index 236f269fa..33b9c9e30 100644 --- a/src/pipecat/services/anthropic/llm.py +++ b/src/pipecat/services/anthropic/llm.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Anthropic AI service integration for Pipecat. + +This module provides LLM services and context management for Anthropic's Claude models, +including support for function calling, vision, and prompt caching features. +""" + import asyncio import base64 import copy @@ -46,6 +52,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.tracing.service_decorators import traced_llm try: @@ -58,27 +65,59 @@ except ModuleNotFoundError as e: @dataclass class AnthropicContextAggregatorPair: + """Pair of context aggregators for Anthropic conversations. + + Encapsulates both user and assistant context aggregators + to manage conversation flow and message formatting. + + Parameters: + _user: The user context aggregator. + _assistant: The assistant context aggregator. + """ + _user: "AnthropicUserContextAggregator" _assistant: "AnthropicAssistantContextAggregator" def user(self) -> "AnthropicUserContextAggregator": + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ return self._user def assistant(self) -> "AnthropicAssistantContextAggregator": + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ return self._assistant class AnthropicLLMService(LLMService): - """This class implements inference with Anthropic's AI models. + """LLM service for Anthropic's Claude models. - Can provide a custom client via the `client` kwarg, allowing you to - use `AsyncAnthropicBedrock` and `AsyncAnthropicVertex` clients + Provides inference capabilities with Claude models including support for + function calling, vision processing, streaming responses, and prompt caching. + Can use custom clients like AsyncAnthropicBedrock and AsyncAnthropicVertex. """ # Overriding the default adapter to use the Anthropic one. adapter_class = AnthropicLLMAdapter class InputParams(BaseModel): + """Input parameters for Anthropic model inference. + + Parameters: + enable_prompt_caching_beta: Whether to enable beta prompt caching feature. + max_tokens: Maximum tokens to generate. Must be at least 1. + temperature: Sampling temperature between 0.0 and 1.0. + top_k: Top-k sampling parameter. + top_p: Top-p sampling parameter between 0.0 and 1.0. + extra: Additional parameters to pass to the API. + """ + enable_prompt_caching_beta: Optional[bool] = False max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1) temperature: Optional[float] = Field(default_factory=lambda: NOT_GIVEN, ge=0.0, le=1.0) @@ -95,6 +134,15 @@ class AnthropicLLMService(LLMService): client=None, **kwargs, ): + """Initialize the Anthropic LLM service. + + Args: + api_key: Anthropic API key for authentication. + model: Model name to use. Defaults to "claude-sonnet-4-20250514". + params: Optional model parameters for inference. + client: Optional custom Anthropic client instance. + **kwargs: Additional arguments passed to parent LLMService. + """ super().__init__(**kwargs) params = params or AnthropicLLMService.InputParams() self._client = client or AsyncAnthropic( @@ -111,10 +159,20 @@ class AnthropicLLMService(LLMService): } def can_generate_metrics(self) -> bool: + """Check if this service can generate usage metrics. + + Returns: + True, as Anthropic provides detailed token usage metrics. + """ return True @property def enable_prompt_caching_beta(self) -> bool: + """Check if prompt caching beta feature is enabled. + + Returns: + True if prompt caching is enabled. + """ return self._enable_prompt_caching_beta def create_context_aggregator( @@ -124,22 +182,19 @@ class AnthropicLLMService(LLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> AnthropicContextAggregatorPair: - """Create an instance of AnthropicContextAggregatorPair from an - OpenAILLMContext. Constructor keyword arguments for both the user and - assistant aggregators can be provided. + """Create Anthropic-specific context aggregators. + + Creates a pair of context aggregators optimized for Anthropic's message format, + including support for function calls, tool usage, and image handling. Args: - context (OpenAILLMContext): The LLM context. - user_params (LLMUserAggregatorParams, optional): User aggregator - parameters. - assistant_params (LLMAssistantAggregatorParams, optional): User - aggregator parameters. + context: The LLM context. + user_params: User aggregator parameters. + assistant_params: Assistant aggregator parameters. Returns: - AnthropicContextAggregatorPair: A pair of context aggregators, one - for the user and one for the assistant, encapsulated in an - AnthropicContextAggregatorPair. - + A pair of context aggregators, one for the user and one for the assistant, + encapsulated in an AnthropicContextAggregatorPair. """ context.set_llm_adapter(self.get_llm_adapter()) @@ -203,7 +258,7 @@ class AnthropicLLMService(LLMService): json_accumulator = "" function_calls = [] - async for event in response: + async for event in WatchdogAsyncIterator(response, manager=self.task_manager): # Aggregate streaming content, create frames, trigger events if event.type == "content_block_delta": @@ -307,6 +362,15 @@ class AnthropicLLMService(LLMService): ) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and route them appropriately. + + Handles various frame types including context frames, message frames, + vision frames, and settings updates. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) context = None @@ -358,6 +422,13 @@ class AnthropicLLMService(LLMService): class AnthropicLLMContext(OpenAILLMContext): + """LLM context specialized for Anthropic's message format and features. + + Extends OpenAILLMContext to handle Anthropic-specific features like + system messages, prompt caching, and message format conversions. + Manages conversation state and message history formatting. + """ + def __init__( self, messages: Optional[List[dict]] = None, @@ -366,6 +437,14 @@ class AnthropicLLMContext(OpenAILLMContext): *, system: Union[str, NotGiven] = NOT_GIVEN, ): + """Initialize the Anthropic LLM context. + + Args: + messages: Initial list of conversation messages. + tools: Available function calling tools. + tool_choice: Tool selection preference. + system: System message content. + """ super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) # For beta prompt caching. This is a counter that tracks the number of turns @@ -378,6 +457,16 @@ class AnthropicLLMContext(OpenAILLMContext): @staticmethod def upgrade_to_anthropic(obj: OpenAILLMContext) -> "AnthropicLLMContext": + """Upgrade an OpenAI context to Anthropic format. + + Converts message format and restructures content for Anthropic compatibility. + + Args: + obj: The OpenAI context to upgrade. + + Returns: + The upgraded Anthropic context. + """ logger.debug(f"Upgrading to Anthropic: {obj}") if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AnthropicLLMContext): obj.__class__ = AnthropicLLMContext @@ -386,6 +475,14 @@ class AnthropicLLMContext(OpenAILLMContext): @classmethod def from_openai_context(cls, openai_context: OpenAILLMContext): + """Create Anthropic context from OpenAI context. + + Args: + openai_context: The OpenAI context to convert. + + Returns: + New Anthropic context with converted messages. + """ self = cls( messages=openai_context.messages, tools=openai_context.tools, @@ -397,12 +494,28 @@ class AnthropicLLMContext(OpenAILLMContext): @classmethod def from_messages(cls, messages: List[dict]) -> "AnthropicLLMContext": + """Create context from a list of messages. + + Args: + messages: List of conversation messages. + + Returns: + New Anthropic context with the provided messages. + """ self = cls(messages=messages) self._restructure_from_openai_messages() return self @classmethod def from_image_frame(cls, frame: VisionImageRawFrame) -> "AnthropicLLMContext": + """Create context from a vision image frame. + + Args: + frame: The vision image frame to process. + + Returns: + New Anthropic context with the image message. + """ context = cls() context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image, text=frame.text @@ -410,11 +523,15 @@ class AnthropicLLMContext(OpenAILLMContext): return context def set_messages(self, messages: List): + """Set the messages list and reset cache tracking. + + Args: + messages: New list of messages to set. + """ self.turns_above_cache_threshold = 0 self._messages[:] = messages self._restructure_from_openai_messages() - # convert a message in Anthropic format into one or more messages in OpenAI format def to_standard_messages(self, obj): """Convert Anthropic message format to standard structured format. @@ -555,6 +672,17 @@ class AnthropicLLMContext(OpenAILLMContext): def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: str = None ): + """Add an image message to the context. + + Converts the image to base64 JPEG format and adds it as a user message + with optional accompanying text. + + Args: + format: The image format (e.g., 'RGB', 'RGBA'). + size: Image dimensions as (width, height). + image: Raw image bytes. + text: Optional text to accompany the image. + """ buffer = io.BytesIO() Image.frombytes(format, size, image).save(buffer, format="JPEG") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") @@ -575,6 +703,14 @@ class AnthropicLLMContext(OpenAILLMContext): self.add_message({"role": "user", "content": content}) def add_message(self, message): + """Add a message to the context, merging with previous message if same role. + + Anthropic requires alternating roles, so consecutive messages from the same + role are merged together. + + Args: + message: The message to add to the context. + """ try: if self.messages: # Anthropic requires that roles alternate. If this message's role is the same as the @@ -600,6 +736,14 @@ class AnthropicLLMContext(OpenAILLMContext): logger.error(f"Error adding message: {e}") def get_messages_with_cache_control_markers(self) -> List[dict]: + """Get messages with prompt caching markers applied. + + Adds cache control markers to appropriate messages based on the + number of turns above the cache threshold. + + Returns: + List of messages with cache control markers added. + """ try: messages = copy.deepcopy(self.messages) if self.turns_above_cache_threshold >= 1 and messages[-1]["role"] == "user": @@ -667,12 +811,26 @@ class AnthropicLLMContext(OpenAILLMContext): message["content"] = [{"type": "text", "text": "(empty)"}] def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Includes system message at the beginning if present. + + Returns: + List of messages suitable for storage. + """ messages = super().get_messages_for_persistent_storage() if self.system: messages.insert(0, {"role": "system", "content": self.system}) return messages def get_messages_for_logging(self) -> str: + """Get messages formatted for logging with sensitive data redacted. + + Replaces image data with placeholder text for cleaner logs. + + Returns: + JSON string representation of messages for logging. + """ msgs = [] for message in self.messages: msg = copy.deepcopy(message) @@ -686,6 +844,12 @@ class AnthropicLLMContext(OpenAILLMContext): class AnthropicUserContextAggregator(LLMUserContextAggregator): + """Anthropic-specific user context aggregator. + + Handles aggregation of user messages for Anthropic LLM services. + Inherits all functionality from the base LLMUserContextAggregator. + """ + pass @@ -700,7 +864,20 @@ class AnthropicUserContextAggregator(LLMUserContextAggregator): class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): + """Context aggregator for assistant messages in Anthropic conversations. + + Handles function call lifecycle management including in-progress tracking, + result handling, and cancellation for Anthropic's tool use format. + """ + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + """Handle a function call that is starting. + + Creates tool use message and placeholder tool result for tracking. + + Args: + frame: Frame containing function call details. + """ assistant_message = {"role": "assistant", "content": []} assistant_message["content"].append( { @@ -725,6 +902,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle the result of a completed function call. + + Updates the tool result with actual return value or completion status. + + Args: + frame: Frame containing function call result. + """ if frame.result: result = json.dumps(frame.result) await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) @@ -734,6 +918,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + """Handle cancellation of a function call. + + Updates the tool result to indicate cancellation. + + Args: + frame: Frame containing function call cancellation details. + """ await self._update_function_call_result( frame.function_name, frame.tool_call_id, "CANCELLED" ) @@ -752,6 +943,14 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): content["content"] = result async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle a user image frame with function call context. + + Marks the associated function call as completed and adds the image + to the conversation context. + + Args: + frame: User image frame with request context. + """ await self._update_function_call_result( frame.request.function_name, frame.request.tool_call_id, "COMPLETED" ) diff --git a/src/pipecat/services/assemblyai/models.py b/src/pipecat/services/assemblyai/models.py index 58b69fdf5..b34ec554d 100644 --- a/src/pipecat/services/assemblyai/models.py +++ b/src/pipecat/services/assemblyai/models.py @@ -1,10 +1,30 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""AssemblyAI WebSocket API message models and connection parameters. + +This module defines Pydantic models for handling AssemblyAI's real-time +transcription WebSocket messages and connection configuration. +""" + from typing import List, Literal, Optional from pydantic import BaseModel, Field class Word(BaseModel): - """Represents a single word in a transcription with timing and confidence.""" + """Represents a single word in a transcription with timing and confidence. + + Parameters: + start: Start time of the word in milliseconds. + end: End time of the word in milliseconds. + text: The transcribed word text. + confidence: Confidence score for the word (0.0 to 1.0). + word_is_final: Whether this word is finalized and won't change. + """ start: int end: int @@ -14,13 +34,23 @@ class Word(BaseModel): class BaseMessage(BaseModel): - """Base class for all AssemblyAI WebSocket messages.""" + """Base class for all AssemblyAI WebSocket messages. + + Parameters: + type: The message type identifier. + """ type: str class BeginMessage(BaseMessage): - """Message sent when a new session begins.""" + """Message sent when a new session begins. + + Parameters: + type: Always "Begin" for this message type. + id: Unique session identifier. + expires_at: Unix timestamp when the session expires. + """ type: Literal["Begin"] = "Begin" id: str @@ -28,7 +58,17 @@ class BeginMessage(BaseMessage): class TurnMessage(BaseMessage): - """Message containing transcription data for a turn of speech.""" + """Message containing transcription data for a turn of speech. + + Parameters: + type: Always "Turn" for this message type. + turn_order: Sequential number of this turn in the session. + turn_is_formatted: Whether the transcript has been formatted. + end_of_turn: Whether this marks the end of a speaking turn. + transcript: The transcribed text for this turn. + end_of_turn_confidence: Confidence score for end-of-turn detection. + words: List of individual words with timing and confidence data. + """ type: Literal["Turn"] = "Turn" turn_order: int @@ -40,7 +80,13 @@ class TurnMessage(BaseMessage): class TerminationMessage(BaseMessage): - """Message sent when the session is terminated.""" + """Message sent when the session is terminated. + + Parameters: + type: Always "Termination" for this message type. + audio_duration_seconds: Total duration of audio processed. + session_duration_seconds: Total duration of the session. + """ type: Literal["Termination"] = "Termination" audio_duration_seconds: float @@ -52,6 +98,18 @@ AnyMessage = BeginMessage | TurnMessage | TerminationMessage class AssemblyAIConnectionParams(BaseModel): + """Configuration parameters for AssemblyAI WebSocket connection. + + Parameters: + sample_rate: Audio sample rate in Hz. Defaults to 16000. + encoding: Audio encoding format. Defaults to "pcm_s16le". + formatted_finals: Whether to enable transcript formatting. Defaults to True. + word_finalization_max_wait_time: Maximum time to wait for word finalization in milliseconds. + end_of_turn_confidence_threshold: Confidence threshold for end-of-turn detection. + min_end_of_turn_silence_when_confident: Minimum silence duration when confident about end-of-turn. + max_turn_silence: Maximum silence duration before forcing end-of-turn. + """ + sample_rate: int = 16000 encoding: Literal["pcm_s16le", "pcm_mulaw"] = "pcm_s16le" formatted_finals: bool = True diff --git a/src/pipecat/services/assemblyai/stt.py b/src/pipecat/services/assemblyai/stt.py index 09aaa4d25..0e7103885 100644 --- a/src/pipecat/services/assemblyai/stt.py +++ b/src/pipecat/services/assemblyai/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AssemblyAI speech-to-text service implementation. + +This module provides integration with AssemblyAI's real-time speech-to-text +WebSocket API for streaming audio transcription. +""" + import asyncio import json from typing import Any, AsyncGenerator, Dict @@ -45,6 +51,13 @@ except ModuleNotFoundError as e: class AssemblyAISTTService(STTService): + """AssemblyAI real-time speech-to-text service. + + Provides real-time speech transcription using AssemblyAI's WebSocket API. + Supports both interim and final transcriptions with configurable parameters + for audio processing and connection management. + """ + def __init__( self, *, @@ -55,6 +68,16 @@ class AssemblyAISTTService(STTService): vad_force_turn_endpoint: bool = True, **kwargs, ): + """Initialize the AssemblyAI STT service. + + Args: + api_key: AssemblyAI API key for authentication. + language: Language code for transcription. Defaults to English (Language.EN). + api_endpoint_base_url: WebSocket endpoint URL. Defaults to AssemblyAI's streaming endpoint. + connection_params: Connection configuration parameters. Defaults to AssemblyAIConnectionParams(). + vad_force_turn_endpoint: Whether to force turn endpoint on VAD stop. Defaults to True. + **kwargs: Additional arguments passed to parent STTService class. + """ self._api_key = api_key self._language = language self._api_endpoint_base_url = api_endpoint_base_url @@ -75,22 +98,50 @@ class AssemblyAISTTService(STTService): self._chunk_size_bytes = 0 def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True if metrics generation is supported. + """ return True async def start(self, frame: StartFrame): + """Start the speech-to-text service. + + Args: + frame: Start frame to begin processing. + """ await super().start(frame) self._chunk_size_bytes = int(self._chunk_size_ms * self._sample_rate * 2 / 1000) await self._connect() async def stop(self, frame: EndFrame): + """Stop the speech-to-text service. + + Args: + frame: End frame to stop processing. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the speech-to-text service. + + Args: + frame: Cancel frame to abort processing. + """ await super().cancel(frame) await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text conversion. + + Args: + audio: Raw audio bytes to process. + + Yields: + None (processing handled via WebSocket messages). + """ self._audio_buffer.extend(audio) while len(self._audio_buffer) >= self._chunk_size_bytes: @@ -101,6 +152,12 @@ class AssemblyAISTTService(STTService): yield None async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for VAD and metrics handling. + + Args: + frame: Frame to process. + direction: Direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): await self.start_ttfb_metrics() @@ -189,17 +246,16 @@ class AssemblyAISTTService(STTService): try: while self._connected: try: - message = await self._websocket.recv() - self.start_watchdog() + message = await asyncio.wait_for(self._websocket.recv(), timeout=1.0) data = json.loads(message) await self._handle_message(data) + except asyncio.TimeoutError: + self.reset_watchdog() except websockets.exceptions.ConnectionClosedOK: break except Exception as e: logger.error(f"Error processing WebSocket message: {e}") break - finally: - self.reset_watchdog() except Exception as e: logger.error(f"Fatal error in receive handler: {e}") diff --git a/src/pipecat/services/aws/llm.py b/src/pipecat/services/aws/llm.py index dcec91463..283cea601 100644 --- a/src/pipecat/services/aws/llm.py +++ b/src/pipecat/services/aws/llm.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Bedrock integration for Large Language Model services. + +This module provides AWS Bedrock LLM service implementation with support for +Amazon Nova and Anthropic Claude models, including vision capabilities and +function calling. +""" + import asyncio import base64 import copy @@ -61,17 +68,44 @@ except ModuleNotFoundError as e: @dataclass class AWSBedrockContextAggregatorPair: + """Container for AWS Bedrock context aggregators. + + Provides convenient access to both user and assistant context aggregators + for AWS Bedrock LLM operations. + + Parameters: + _user: The user context aggregator instance. + _assistant: The assistant context aggregator instance. + """ + _user: "AWSBedrockUserContextAggregator" _assistant: "AWSBedrockAssistantContextAggregator" def user(self) -> "AWSBedrockUserContextAggregator": + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ return self._user def assistant(self) -> "AWSBedrockAssistantContextAggregator": + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ return self._assistant class AWSBedrockLLMContext(OpenAILLMContext): + """AWS Bedrock-specific LLM context implementation. + + Extends OpenAI LLM context to handle AWS Bedrock's specific message format + and system message handling. Manages conversion between OpenAI and Bedrock + message formats. + """ + def __init__( self, messages: Optional[List[dict]] = None, @@ -80,11 +114,27 @@ class AWSBedrockLLMContext(OpenAILLMContext): *, system: Optional[str] = None, ): + """Initialize AWS Bedrock LLM context. + + Args: + messages: List of conversation messages in OpenAI format. + tools: List of available function calling tools. + tool_choice: Tool selection strategy or specific tool choice. + system: System message content for AWS Bedrock. + """ super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) self.system = system @staticmethod def upgrade_to_bedrock(obj: OpenAILLMContext) -> "AWSBedrockLLMContext": + """Upgrade an OpenAI LLM context to AWS Bedrock format. + + Args: + obj: The OpenAI LLM context to upgrade. + + Returns: + The upgraded AWS Bedrock LLM context. + """ logger.debug(f"Upgrading to AWS Bedrock: {obj}") if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSBedrockLLMContext): obj.__class__ = AWSBedrockLLMContext @@ -95,6 +145,14 @@ class AWSBedrockLLMContext(OpenAILLMContext): @classmethod def from_openai_context(cls, openai_context: OpenAILLMContext): + """Create AWS Bedrock context from OpenAI context. + + Args: + openai_context: The OpenAI LLM context to convert. + + Returns: + New AWS Bedrock LLM context instance. + """ self = cls( messages=openai_context.messages, tools=openai_context.tools, @@ -106,12 +164,28 @@ class AWSBedrockLLMContext(OpenAILLMContext): @classmethod def from_messages(cls, messages: List[dict]) -> "AWSBedrockLLMContext": + """Create AWS Bedrock context from message list. + + Args: + messages: List of messages in OpenAI format. + + Returns: + New AWS Bedrock LLM context instance. + """ self = cls(messages=messages) self._restructure_from_openai_messages() return self @classmethod def from_image_frame(cls, frame: VisionImageRawFrame) -> "AWSBedrockLLMContext": + """Create AWS Bedrock context from vision image frame. + + Args: + frame: The vision image frame to convert. + + Returns: + New AWS Bedrock LLM context instance. + """ context = cls() context.add_image_frame_message( format=frame.format, size=frame.size, image=frame.image, text=frame.text @@ -119,10 +193,14 @@ class AWSBedrockLLMContext(OpenAILLMContext): return context def set_messages(self, messages: List): + """Set the messages list and restructure for Bedrock format. + + Args: + messages: List of messages to set. + """ self._messages[:] = messages self._restructure_from_openai_messages() - # convert a message in AWS Bedrock format into one or more messages in OpenAI format def to_standard_messages(self, obj): """Convert AWS Bedrock message format to standard structured format. @@ -295,6 +373,14 @@ class AWSBedrockLLMContext(OpenAILLMContext): def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: str = None ): + """Add an image message to the context. + + Args: + format: The image format (e.g., 'RGB', 'RGBA'). + size: The image dimensions as (width, height). + image: The raw image data as bytes. + text: Optional text to accompany the image. + """ buffer = io.BytesIO() Image.frombytes(format, size, image).save(buffer, format="JPEG") encoded_image = base64.b64encode(buffer.getvalue()).decode("utf-8") @@ -306,6 +392,14 @@ class AWSBedrockLLMContext(OpenAILLMContext): self.add_message({"role": "user", "content": content}) def add_message(self, message): + """Add a message to the context, merging with previous message if same role. + + AWS Bedrock requires alternating roles, so consecutive messages from the + same role are merged together. + + Args: + message: The message to add to the context. + """ try: if self.messages: # AWS Bedrock requires that roles alternate. If this message's @@ -330,10 +424,10 @@ class AWSBedrockLLMContext(OpenAILLMContext): logger.error(f"Error adding message: {e}") def _restructure_from_bedrock_messages(self): - """Restructure messages in AWS Bedrock format by handling system - messages, merging consecutive messages with the same role, and ensuring - proper content formatting. + """Restructure messages in AWS Bedrock format. + Handles system messages, merging consecutive messages with the same role, + and ensuring proper content formatting. """ # Handle system message if present at the beginning if self.messages and self.messages[0]["role"] == "system": @@ -416,12 +510,22 @@ class AWSBedrockLLMContext(OpenAILLMContext): message["content"] = [{"type": "text", "text": "(empty)"}] def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Returns: + List of messages including system message if present. + """ messages = super().get_messages_for_persistent_storage() if self.system: messages.insert(0, {"role": "system", "content": self.system}) return messages def get_messages_for_logging(self) -> str: + """Get messages formatted for logging with sensitive data redacted. + + Returns: + JSON string representation of messages with image data redacted. + """ msgs = [] for message in self.messages: msg = copy.deepcopy(message) @@ -435,11 +539,36 @@ class AWSBedrockLLMContext(OpenAILLMContext): class AWSBedrockUserContextAggregator(LLMUserContextAggregator): + """User context aggregator for AWS Bedrock LLM service. + + Handles aggregation of user messages and frames for AWS Bedrock format. + Inherits all functionality from the base LLM user context aggregator. + + Args: + context: The LLM context to aggregate messages into. + params: Configuration parameters for the aggregator. + """ + pass class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator): + """Assistant context aggregator for AWS Bedrock LLM service. + + Handles aggregation of assistant responses and function calls for AWS Bedrock + format, including tool use and tool result handling. + + Args: + context: The LLM context to aggregate messages into. + params: Configuration parameters for the aggregator. + """ + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + """Handle function call in progress frame. + + Args: + frame: The function call in progress frame to handle. + """ # Format tool use according to AWS Bedrock API self._context.add_message( { @@ -470,6 +599,11 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call result frame. + + Args: + frame: The function call result frame to handle. + """ if frame.result: result = json.dumps(frame.result) await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) @@ -479,6 +613,11 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + """Handle function call cancel frame. + + Args: + frame: The function call cancel frame to handle. + """ await self._update_function_call_result( frame.function_name, frame.tool_call_id, "CANCELLED" ) @@ -497,6 +636,11 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator): content["toolResult"]["content"] = [{"text": result}] async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle user image frame. + + Args: + frame: The user image frame to handle. + """ await self._update_function_call_result( frame.request.function_name, frame.request.tool_call_id, "COMPLETED" ) @@ -509,18 +653,28 @@ class AWSBedrockAssistantContextAggregator(LLMAssistantContextAggregator): class AWSBedrockLLMService(LLMService): - """This class implements inference with AWS Bedrock models including Amazon - Nova and Anthropic Claude. - - Requires AWS credentials to be configured in the environment or through - boto3 configuration. + """AWS Bedrock Large Language Model service implementation. + Provides inference capabilities for AWS Bedrock models including Amazon Nova + and Anthropic Claude. Supports streaming responses, function calling, and + vision capabilities. """ # Overriding the default adapter to use the Anthropic one. adapter_class = AWSBedrockLLMAdapter class InputParams(BaseModel): + """Input parameters for AWS Bedrock LLM service. + + Parameters: + max_tokens: Maximum number of tokens to generate. + temperature: Sampling temperature between 0.0 and 1.0. + top_p: Nucleus sampling parameter between 0.0 and 1.0. + stop_sequences: List of strings that stop generation. + latency: Performance mode - "standard" or "optimized". + additional_model_request_fields: Additional model-specific parameters. + """ + max_tokens: Optional[int] = Field(default_factory=lambda: 4096, ge=1) temperature: Optional[float] = Field(default_factory=lambda: 0.7, ge=0.0, le=1.0) top_p: Optional[float] = Field(default_factory=lambda: 0.999, ge=0.0, le=1.0) @@ -540,6 +694,18 @@ class AWSBedrockLLMService(LLMService): client_config: Optional[Config] = None, **kwargs, ): + """Initialize the AWS Bedrock LLM service. + + Args: + model: The AWS Bedrock model identifier to use. + aws_access_key: AWS access key ID. If None, uses default credentials. + aws_secret_key: AWS secret access key. If None, uses default credentials. + aws_session_token: AWS session token for temporary credentials. + aws_region: AWS region for the Bedrock service. + params: Model parameters and configuration. + client_config: Custom boto3 client configuration. + **kwargs: Additional arguments passed to parent LLMService. + """ super().__init__(**kwargs) params = params or AWSBedrockLLMService.InputParams() @@ -573,6 +739,11 @@ class AWSBedrockLLMService(LLMService): logger.info(f"Using AWS Bedrock model: {model}") def can_generate_metrics(self) -> bool: + """Check if the service can generate usage metrics. + + Returns: + True if metrics generation is supported. + """ return True def create_context_aggregator( @@ -582,21 +753,21 @@ class AWSBedrockLLMService(LLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> AWSBedrockContextAggregatorPair: - """Create an instance of AWSBedrockContextAggregatorPair from an - OpenAILLMContext. Constructor keyword arguments for both the user and - assistant aggregators can be provided. + """Create AWS Bedrock-specific context aggregators. + + Creates a pair of context aggregators optimized for AWS Bedrocks's message + format, including support for function calls, tool usage, and image handling. Args: - context (OpenAILLMContext): The LLM context. - user_params (LLMUserAggregatorParams, optional): User aggregator - parameters. - assistant_params (LLMAssistantAggregatorParams, optional): User - aggregator parameters. + context: The LLM context to create aggregators for. + user_params: Parameters for user message aggregation. + assistant_params: Parameters for assistant message aggregation. Returns: - AWSBedrockContextAggregatorPair: A pair of context aggregators, one - for the user and one for the assistant, encapsulated in an + AWSBedrockContextAggregatorPair: A pair of context aggregators, one for + the user and one for the assistant, encapsulated in an AWSBedrockContextAggregatorPair. + """ context.set_llm_adapter(self.get_llm_adapter()) @@ -711,6 +882,8 @@ class AWSBedrockLLMService(LLMService): function_calls = [] for event in response["stream"]: + self.reset_watchdog() + # Handle text content if "contentBlockDelta" in event: delta = event["contentBlockDelta"]["delta"] @@ -762,6 +935,7 @@ class AWSBedrockLLMService(LLMService): completion_tokens += usage.get("outputTokens", 0) cache_read_input_tokens += usage.get("cacheReadInputTokens", 0) cache_creation_input_tokens += usage.get("cacheWriteInputTokens", 0) + await self.run_function_calls(function_calls) except asyncio.CancelledError: # If we're interrupted, we won't get a complete usage report. So set our flag to use the @@ -789,6 +963,12 @@ class AWSBedrockLLMService(LLMService): ) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle LLM-specific frame types. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) context = None diff --git a/src/pipecat/services/aws/stt.py b/src/pipecat/services/aws/stt.py index c9c1b32d1..a7f8fea97 100644 --- a/src/pipecat/services/aws/stt.py +++ b/src/pipecat/services/aws/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Transcribe Speech-to-Text service implementation. + +This module provides a WebSocket-based connection to AWS Transcribe for real-time +speech-to-text transcription with support for multiple languages and audio formats. +""" + import asyncio import json import os @@ -37,6 +43,13 @@ except ModuleNotFoundError as e: class AWSTranscribeSTTService(STTService): + """AWS Transcribe Speech-to-Text service using WebSocket streaming. + + Provides real-time speech transcription using AWS Transcribe's streaming API. + Supports multiple languages, configurable sample rates, and both interim and + final transcription results. + """ + def __init__( self, *, @@ -48,6 +61,17 @@ class AWSTranscribeSTTService(STTService): language: Language = Language.EN, **kwargs, ): + """Initialize the AWS Transcribe STT service. + + Args: + api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable. + aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable. + aws_session_token: AWS session token for temporary credentials. If None, uses AWS_SESSION_TOKEN environment variable. + region: AWS region for the service. Defaults to "us-east-1". + sample_rate: Audio sample rate in Hz. Must be 8000 or 16000. Defaults to 16000. + language: Language for transcription. Defaults to English. + **kwargs: Additional arguments passed to parent STTService class. + """ super().__init__(**kwargs) self._settings = { @@ -79,14 +103,28 @@ class AWSTranscribeSTTService(STTService): self._receive_task = None def get_service_encoding(self, encoding: str) -> str: - """Convert internal encoding format to AWS Transcribe format.""" + """Convert internal encoding format to AWS Transcribe format. + + Args: + encoding: Internal encoding format string. + + Returns: + AWS Transcribe compatible encoding format. + """ encoding_map = { "linear16": "pcm", # AWS expects "pcm" for 16-bit linear PCM } return encoding_map.get(encoding, encoding) async def start(self, frame: StartFrame): - """Initialize the connection when the service starts.""" + """Initialize the connection when the service starts. + + Args: + frame: Start frame signaling service initialization. + + Raises: + RuntimeError: If WebSocket connection cannot be established after retries. + """ await super().start(frame) logger.info("Starting AWS Transcribe service...") retry_count = 0 @@ -108,15 +146,32 @@ class AWSTranscribeSTTService(STTService): raise RuntimeError("Failed to establish WebSocket connection after multiple attempts") async def stop(self, frame: EndFrame): + """Stop the service and disconnect from AWS Transcribe. + + Args: + frame: End frame signaling service shutdown. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the service and disconnect from AWS Transcribe. + + Args: + frame: Cancel frame signaling service cancellation. + """ await super().cancel(frame) await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Process audio data and send to AWS Transcribe""" + """Process audio data and send to AWS Transcribe. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + ErrorFrame: If processing fails or connection issues occur. + """ try: # Ensure WebSocket is connected if not self._ws_client or not self._ws_client.open: @@ -255,7 +310,14 @@ class AWSTranscribeSTTService(STTService): self._ws_client = None def language_to_service_language(self, language: Language) -> str | None: - """Convert internal language enum to AWS Transcribe language code.""" + """Convert internal language enum to AWS Transcribe language code. + + Args: + language: Internal language enumeration value. + + Returns: + AWS Transcribe compatible language code, or None if unsupported. + """ language_map = { Language.EN: "en-US", Language.ES: "es-US", @@ -284,9 +346,7 @@ class AWSTranscribeSTTService(STTService): break try: - response = await self._ws_client.recv() - - self.start_watchdog() + response = await asyncio.wait_for(self._ws_client.recv(), timeout=1.0) headers, payload = decode_event(response) @@ -337,6 +397,8 @@ class AWSTranscribeSTTService(STTService): else: logger.debug(f"{self} Other message type received: {headers}") logger.debug(f"{self} Payload: {payload}") + except asyncio.TimeoutError: + self.reset_watchdog() except websockets.exceptions.ConnectionClosed as e: logger.error( f"{self} WebSocket connection closed in receive loop with code {e.code}: {e.reason}" @@ -345,5 +407,3 @@ class AWSTranscribeSTTService(STTService): except Exception as e: logger.error(f"{self} Unexpected error in receive loop: {e}") break - finally: - self.reset_watchdog() diff --git a/src/pipecat/services/aws/tts.py b/src/pipecat/services/aws/tts.py index 762e8b9e4..ce89dea9e 100644 --- a/src/pipecat/services/aws/tts.py +++ b/src/pipecat/services/aws/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Polly text-to-speech service implementation. + +This module provides integration with Amazon Polly for text-to-speech synthesis, +supporting multiple languages, voices, and SSML features. +""" + import asyncio import os from typing import AsyncGenerator, List, Optional @@ -33,6 +39,14 @@ except ModuleNotFoundError as e: def language_to_aws_language(language: Language) -> Optional[str]: + """Convert a Language enum to AWS Polly language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding AWS Polly language code, or None if not supported. + """ language_map = { # Arabic Language.AR: "arb", @@ -109,7 +123,25 @@ def language_to_aws_language(language: Language) -> Optional[str]: class AWSPollyTTSService(TTSService): + """AWS Polly text-to-speech service. + + Provides text-to-speech synthesis using Amazon Polly with support for + multiple languages, voices, SSML features, and voice customization + options including prosody controls. + """ + class InputParams(BaseModel): + """Input parameters for AWS Polly TTS configuration. + + Parameters: + engine: TTS engine to use ('standard', 'neural', etc.). + language: Language for synthesis. Defaults to English. + pitch: Voice pitch adjustment (for standard engine only). + rate: Speech rate adjustment. + volume: Voice volume adjustment. + lexicon_names: List of pronunciation lexicons to apply. + """ + engine: Optional[str] = None language: Optional[Language] = Language.EN pitch: Optional[str] = None @@ -129,6 +161,18 @@ class AWSPollyTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initializes the AWS Polly TTS service. + + Args: + api_key: AWS secret access key. If None, uses AWS_SECRET_ACCESS_KEY environment variable. + aws_access_key_id: AWS access key ID. If None, uses AWS_ACCESS_KEY_ID environment variable. + aws_session_token: AWS session token for temporary credentials. + region: AWS region for Polly service. Defaults to 'us-east-1'. + voice_id: Voice ID to use for synthesis. Defaults to 'Joanna'. + sample_rate: Audio sample rate. If None, uses service default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to parent TTSService class. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or AWSPollyTTSService.InputParams() @@ -174,9 +218,22 @@ class AWSPollyTTSService(TTSService): ) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as AWS Polly service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to AWS Polly language format. + + Args: + language: The language to convert. + + Returns: + The AWS Polly-specific language code, or None if not supported. + """ return language_to_aws_language(language) def _construct_ssml(self, text: str) -> str: @@ -214,6 +271,15 @@ class AWSPollyTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using AWS Polly. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ + def read_audio_data(**args): response = self._polly_client.synthesize_speech(**args) if "AudioStream" in response: @@ -277,7 +343,14 @@ class AWSPollyTTSService(TTSService): class PollyTTSService(AWSPollyTTSService): + """Deprecated alias for AWSPollyTTSService.""" + def __init__(self, **kwargs): + """Initialize the deprecated PollyTTSService. + + Args: + **kwargs: All arguments passed to AWSPollyTTSService. + """ super().__init__(**kwargs) import warnings diff --git a/src/pipecat/services/aws/utils.py b/src/pipecat/services/aws/utils.py index db69456e9..cfd36b417 100644 --- a/src/pipecat/services/aws/utils.py +++ b/src/pipecat/services/aws/utils.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Transcribe utility functions and classes for WebSocket streaming. + +This module provides utilities for creating presigned URLs, building event messages, +and handling AWS event stream protocol for real-time transcription services. +""" + import binascii import datetime import hashlib @@ -29,7 +35,31 @@ def get_presigned_url( show_speaker_label: bool = False, enable_channel_identification: bool = False, ) -> str: - """Create a presigned URL for AWS Transcribe streaming.""" + """Create a presigned URL for AWS Transcribe streaming. + + Args: + region: AWS region for the service. + credentials: Dictionary containing AWS credentials with keys: + - access_key: AWS access key ID + - secret_key: AWS secret access key + - session_token: AWS session token (optional) + language_code: Language code for transcription (e.g., "en-US"). + media_encoding: Audio encoding format. Defaults to "pcm". + sample_rate: Audio sample rate in Hz. Defaults to 16000. + number_of_channels: Number of audio channels. Defaults to 1. + enable_partial_results_stabilization: Whether to enable partial result stabilization. + partial_results_stability: Stability level for partial results. + vocabulary_name: Custom vocabulary name to use. + vocabulary_filter_name: Vocabulary filter name to apply. + show_speaker_label: Whether to include speaker labels. + enable_channel_identification: Whether to enable channel identification. + + Returns: + Presigned WebSocket URL for AWS Transcribe streaming. + + Raises: + ValueError: If required AWS credentials are missing. + """ access_key = credentials.get("access_key") secret_key = credentials.get("secret_key") session_token = credentials.get("session_token") @@ -58,9 +88,23 @@ def get_presigned_url( class AWSTranscribePresignedURL: + """Generator for AWS Transcribe presigned WebSocket URLs. + + Handles AWS Signature Version 4 signing process to create authenticated + WebSocket URLs for streaming transcription requests. + """ + def __init__( self, access_key: str, secret_key: str, session_token: str, region: str = "us-east-1" ): + """Initialize the presigned URL generator. + + Args: + access_key: AWS access key ID. + secret_key: AWS secret access key. + session_token: AWS session token for temporary credentials. + region: AWS region for the service. Defaults to "us-east-1". + """ self.access_key = access_key self.secret_key = secret_key self.session_token = session_token @@ -96,6 +140,23 @@ class AWSTranscribePresignedURL: enable_partial_results_stabilization: bool = False, partial_results_stability: str = "", ) -> str: + """Generate a presigned WebSocket URL for AWS Transcribe. + + Args: + sample_rate: Audio sample rate in Hz. + language_code: Language code for transcription. + media_encoding: Audio encoding format. + vocabulary_name: Custom vocabulary name. + vocabulary_filter_name: Vocabulary filter name. + show_speaker_label: Whether to include speaker labels. + enable_channel_identification: Whether to enable channel identification. + number_of_channels: Number of audio channels. + enable_partial_results_stabilization: Whether to enable partial result stabilization. + partial_results_stability: Stability level for partial results. + + Returns: + Presigned WebSocket URL with authentication parameters. + """ self.endpoint = f"wss://transcribestreaming.{self.region}.amazonaws.com:8443" self.host = f"transcribestreaming.{self.region}.amazonaws.com:8443" @@ -172,7 +233,15 @@ class AWSTranscribePresignedURL: def get_headers(header_name: str, header_value: str) -> bytearray: - """Build a header following AWS event stream format.""" + """Build a header following AWS event stream format. + + Args: + header_name: Name of the header. + header_value: Value of the header. + + Returns: + Encoded header as a bytearray following AWS event stream protocol. + """ name = header_name.encode("utf-8") name_byte_length = bytes([len(name)]) value_type = bytes([7]) # 7 represents a string @@ -190,9 +259,21 @@ def get_headers(header_name: str, header_value: str) -> bytearray: def build_event_message(payload: bytes) -> bytes: - """ - Build an event message for AWS Transcribe streaming. - Matches AWS sample: https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py + """Build an event message for AWS Transcribe streaming. + + Creates a properly formatted AWS event stream message containing audio data + for real-time transcription. Follows the AWS event stream protocol with + prelude, headers, payload, and CRC checksums. + + Args: + payload: Raw audio bytes to include in the event message. + + Returns: + Complete event message as bytes, ready to send via WebSocket. + + Note: + Implementation matches AWS sample: + https://github.com/aws-samples/amazon-transcribe-streaming-python-websockets/blob/main/eventstream.py """ # Build headers content_type_header = get_headers(":content-type", "application/octet-stream") @@ -235,6 +316,22 @@ def build_event_message(payload: bytes) -> bytes: def decode_event(message): + """Decode an AWS event stream message. + + Parses an AWS event stream message to extract headers and payload, + verifying CRC checksums for data integrity. + + Args: + message: Raw event stream message bytes received from AWS. + + Returns: + Tuple containing: + - Dictionary of parsed headers + - Dictionary of parsed JSON payload + + Raises: + AssertionError: If CRC checksum verification fails. + """ # Extract the prelude, headers, payload and CRC prelude = message[:8] total_length, headers_length = struct.unpack(">II", prelude) diff --git a/src/pipecat/services/aws_nova_sonic/aws.py b/src/pipecat/services/aws_nova_sonic/aws.py index 718e3dc50..77e9575b5 100644 --- a/src/pipecat/services/aws_nova_sonic/aws.py +++ b/src/pipecat/services/aws_nova_sonic/aws.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""AWS Nova Sonic LLM service implementation for Pipecat AI framework. + +This module provides a speech-to-speech LLM service using AWS Nova Sonic, which supports +bidirectional audio streaming, text generation, and function calling capabilities. +""" + import asyncio import base64 import json @@ -56,6 +62,7 @@ from pipecat.services.aws_nova_sonic.context import ( ) from pipecat.services.aws_nova_sonic.frames import AWSNovaSonicFunctionCallResultFrame from pipecat.services.llm_service import LLMService +from pipecat.utils.asyncio.watchdog_coroutine import watchdog_coroutine from pipecat.utils.time import time_now_iso8601 try: @@ -83,28 +90,55 @@ except ModuleNotFoundError as e: class AWSNovaSonicUnhandledFunctionException(Exception): + """Exception raised when the LLM attempts to call an unregistered function.""" + pass class ContentType(Enum): + """Content types supported by AWS Nova Sonic. + + Parameters: + AUDIO: Audio content type. + TEXT: Text content type. + TOOL: Tool content type. + """ + AUDIO = "AUDIO" TEXT = "TEXT" TOOL = "TOOL" class TextStage(Enum): + """Text generation stages in AWS Nova Sonic responses. + + Parameters: + FINAL: Final text that has been fully generated. + SPECULATIVE: Speculative text that is still being generated. + """ + FINAL = "FINAL" # what has been said SPECULATIVE = "SPECULATIVE" # what's planned to be said @dataclass class CurrentContent: + """Represents content currently being received from AWS Nova Sonic. + + Parameters: + type: The type of content (audio, text, or tool). + role: The role generating the content (user, assistant, etc.). + text_stage: The stage of text generation (final or speculative). + text_content: The actual text content if applicable. + """ + type: ContentType role: Role text_stage: TextStage # None if not text text_content: str # starts as None, then fills in if text def __str__(self): + """String representation of the current content.""" return ( f"CurrentContent(\n" f" type={self.type.name},\n" @@ -115,6 +149,20 @@ class CurrentContent: class Params(BaseModel): + """Configuration parameters for AWS Nova Sonic. + + Attributes: + input_sample_rate: Audio input sample rate in Hz. + input_sample_size: Audio input sample size in bits. + input_channel_count: Number of input audio channels. + output_sample_rate: Audio output sample rate in Hz. + output_sample_size: Audio output sample size in bits. + output_channel_count: Number of output audio channels. + max_tokens: Maximum number of tokens to generate. + top_p: Nucleus sampling parameter. + temperature: Sampling temperature for text generation. + """ + # Audio input input_sample_rate: Optional[int] = Field(default=16000) input_sample_size: Optional[int] = Field(default=16) @@ -132,6 +180,12 @@ class Params(BaseModel): class AWSNovaSonicLLMService(LLMService): + """AWS Nova Sonic speech-to-speech LLM service. + + Provides bidirectional audio streaming, real-time transcription, text generation, + and function calling capabilities using AWS Nova Sonic model. + """ + # Override the default adapter to use the AWSNovaSonicLLMAdapter one adapter_class = AWSNovaSonicLLMAdapter @@ -149,6 +203,20 @@ class AWSNovaSonicLLMService(LLMService): send_transcription_frames: bool = True, **kwargs, ): + """Initializes the AWS Nova Sonic LLM service. + + Args: + secret_access_key: AWS secret access key for authentication. + access_key_id: AWS access key ID for authentication. + region: AWS region where the service is hosted. + model: Model identifier. Defaults to "amazon.nova-sonic-v1:0". + voice_id: Voice ID for speech synthesis. Options: matthew, tiffany, amy. + params: Model parameters for audio configuration and inference. + system_instruction: System-level instruction for the model. + tools: Available tools/functions for the model to use. + send_transcription_frames: Whether to emit transcription frames. + **kwargs: Additional arguments passed to the parent LLMService. + """ super().__init__(**kwargs) self._secret_access_key = secret_access_key self._access_key_id = access_key_id @@ -188,16 +256,31 @@ class AWSNovaSonicLLMService(LLMService): # async def start(self, frame: StartFrame): + """Start the service and initiate connection to AWS Nova Sonic. + + Args: + frame: The start frame triggering service initialization. + """ await super().start(frame) self._wants_connection = True await self._start_connecting() async def stop(self, frame: EndFrame): + """Stop the service and close connections. + + Args: + frame: The end frame triggering service shutdown. + """ await super().stop(frame) self._wants_connection = False await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the service and close connections. + + Args: + frame: The cancel frame triggering service cancellation. + """ await super().cancel(frame) self._wants_connection = False await self._disconnect() @@ -207,6 +290,11 @@ class AWSNovaSonicLLMService(LLMService): # async def reset_conversation(self): + """Reset the conversation state while preserving context. + + Handles bot stopped speaking event, disconnects from the service, + and reconnects with the preserved context. + """ logger.debug("Resetting conversation") await self._handle_bot_stopped_speaking(delay_to_catch_trailing_assistant_text=False) @@ -222,6 +310,12 @@ class AWSNovaSonicLLMService(LLMService): # async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle service-specific logic. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ await super().process_frame(frame, direction) if isinstance(frame, OpenAILLMContextFrame): @@ -697,9 +791,7 @@ class AWSNovaSonicLLMService(LLMService): try: while self._stream and not self._disconnecting: output = await self._stream.await_output() - result = await output[1].receive() - - self.start_watchdog() + result = await watchdog_coroutine(output[1].receive(), manager=self.task_manager) if result.value and result.value.bytes_: response_data = result.value.bytes_.decode("utf-8") @@ -728,13 +820,10 @@ class AWSNovaSonicLLMService(LLMService): elif "completionEnd" in event_json: # Handle the LLM completion ending await self._handle_completion_end_event(event_json) - except Exception as e: logger.error(f"{self} error processing responses: {e}") if self._wants_connection: await self.reset_conversation() - finally: - self.reset_watchdog() async def _handle_completion_start_event(self, event_json): pass @@ -961,6 +1050,16 @@ class AWSNovaSonicLLMService(LLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> AWSNovaSonicContextAggregatorPair: + """Create context aggregator pair for managing conversation context. + + Args: + context: The OpenAI LLM context to upgrade. + user_params: Parameters for the user context aggregator. + assistant_params: Parameters for the assistant context aggregator. + + Returns: + A pair of user and assistant context aggregators. + """ context.set_llm_adapter(self.get_llm_adapter()) user = AWSNovaSonicUserContextAggregator(context=context, params=user_params) @@ -979,6 +1078,14 @@ class AWSNovaSonicLLMService(LLMService): ) async def trigger_assistant_response(self): + """Trigger an assistant response by sending audio cue. + + Sends a pre-recorded "ready" audio trigger to prompt the assistant + to start speaking. This is useful for controlling conversation flow. + + Returns: + False if already triggering a response, True otherwise. + """ if self._triggering_assistant_response: return False diff --git a/src/pipecat/services/aws_nova_sonic/context.py b/src/pipecat/services/aws_nova_sonic/context.py index 95f330f61..e23a18362 100644 --- a/src/pipecat/services/aws_nova_sonic/context.py +++ b/src/pipecat/services/aws_nova_sonic/context.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Context management for AWS Nova Sonic LLM service. + +This module provides specialized context aggregators and message handling for AWS Nova Sonic, +including conversation history management and role-specific message processing. +""" + import copy from dataclasses import dataclass, field from enum import Enum @@ -35,6 +41,15 @@ from pipecat.services.openai.llm import ( class Role(Enum): + """Roles supported in AWS Nova Sonic conversations. + + Parameters: + SYSTEM: System-level messages (not used in conversation history). + USER: Messages sent by the user. + ASSISTANT: Messages sent by the assistant. + TOOL: Messages sent by tools (not used in conversation history). + """ + SYSTEM = "SYSTEM" USER = "USER" ASSISTANT = "ASSISTANT" @@ -43,18 +58,45 @@ class Role(Enum): @dataclass class AWSNovaSonicConversationHistoryMessage: + """A single message in AWS Nova Sonic conversation history. + + Parameters: + role: The role of the message sender (USER or ASSISTANT only). + text: The text content of the message. + """ + role: Role # only USER and ASSISTANT text: str @dataclass class AWSNovaSonicConversationHistory: + """Complete conversation history for AWS Nova Sonic initialization. + + Parameters: + system_instruction: System-level instruction for the conversation. + messages: List of conversation messages between user and assistant. + """ + system_instruction: str = None messages: list[AWSNovaSonicConversationHistoryMessage] = field(default_factory=list) class AWSNovaSonicLLMContext(OpenAILLMContext): + """Specialized LLM context for AWS Nova Sonic service. + + Extends OpenAI context with Nova Sonic-specific message handling, + conversation history management, and text buffering capabilities. + """ + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize AWS Nova Sonic LLM context. + + Args: + messages: Initial messages for the context. + tools: Available tools for the context. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(messages=messages, tools=tools, **kwargs) self.__setup_local() @@ -67,6 +109,15 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): def upgrade_to_nova_sonic( obj: OpenAILLMContext, system_instruction: str ) -> "AWSNovaSonicLLMContext": + """Upgrade an OpenAI context to AWS Nova Sonic context. + + Args: + obj: The OpenAI context to upgrade. + system_instruction: System instruction for the context. + + Returns: + The upgraded AWS Nova Sonic context. + """ if isinstance(obj, OpenAILLMContext) and not isinstance(obj, AWSNovaSonicLLMContext): obj.__class__ = AWSNovaSonicLLMContext obj.__setup_local(system_instruction) @@ -74,6 +125,14 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): # NOTE: this method has the side-effect of updating _system_instruction from messages def get_messages_for_initializing_history(self) -> AWSNovaSonicConversationHistory: + """Get conversation history for initializing AWS Nova Sonic session. + + Processes stored messages and extracts system instruction and conversation + history in the format expected by AWS Nova Sonic. + + Returns: + Formatted conversation history with system instruction and messages. + """ history = AWSNovaSonicConversationHistory(system_instruction=self._system_instruction) # Bail if there are no messages @@ -103,6 +162,11 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): return history def get_messages_for_persistent_storage(self): + """Get messages formatted for persistent storage. + + Returns: + List of messages including system instruction if present. + """ messages = super().get_messages_for_persistent_storage() # If we have a system instruction and messages doesn't already contain it, add it if self._system_instruction and not (messages and messages[0].get("role") == "system"): @@ -110,6 +174,14 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): return messages def from_standard_message(self, message) -> AWSNovaSonicConversationHistoryMessage: + """Convert standard message format to Nova Sonic format. + + Args: + message: Standard message dictionary to convert. + + Returns: + Nova Sonic conversation history message, or None if not convertible. + """ role = message.get("role") if message.get("role") == "user" or message.get("role") == "assistant": content = message.get("content") @@ -131,10 +203,20 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): # Sonic conversation history def buffer_user_text(self, text): + """Buffer user text for later flushing to context. + + Args: + text: User text to buffer. + """ self._user_text += f" {text}" if self._user_text else text # logger.debug(f"User text buffered: {self._user_text}") def flush_aggregated_user_text(self) -> str: + """Flush buffered user text to context as a complete message. + + Returns: + The flushed user text, or empty string if no text was buffered. + """ if not self._user_text: return "" user_text = self._user_text @@ -148,10 +230,16 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): return user_text def buffer_assistant_text(self, text): + """Buffer assistant text for later flushing to context. + + Args: + text: Assistant text to buffer. + """ self._assistant_text += text # logger.debug(f"Assistant text buffered: {self._assistant_text}") def flush_aggregated_assistant_text(self): + """Flush buffered assistant text to context as a complete message.""" if not self._assistant_text: return message = { @@ -165,13 +253,31 @@ class AWSNovaSonicLLMContext(OpenAILLMContext): @dataclass class AWSNovaSonicMessagesUpdateFrame(DataFrame): + """Frame containing updated AWS Nova Sonic context. + + Parameters: + context: The updated AWS Nova Sonic LLM context. + """ + context: AWSNovaSonicLLMContext class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): + """Context aggregator for user messages in AWS Nova Sonic conversations. + + Extends the OpenAI user context aggregator to emit Nova Sonic-specific + context update frames. + """ + async def process_frame( self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM ): + """Process frames and emit Nova Sonic-specific context updates. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ await super().process_frame(frame, direction) # Parent does not push LLMMessagesUpdateFrame @@ -180,7 +286,19 @@ class AWSNovaSonicUserContextAggregator(OpenAIUserContextAggregator): class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Context aggregator for assistant messages in AWS Nova Sonic conversations. + + Provides specialized handling for assistant responses and function calls + in AWS Nova Sonic context, with custom frame processing logic. + """ + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Nova Sonic-specific logic. + + Args: + frame: The frame to process. + direction: The direction the frame is traveling. + """ # HACK: For now, disable the context aggregator by making it just pass through all frames # that the parent handles (except the function call stuff, which we still need). # For an explanation of this hack, see @@ -205,6 +323,11 @@ class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): await super().process_frame(frame, direction) async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call results for AWS Nova Sonic. + + Args: + frame: The function call result frame to handle. + """ await super().handle_function_call_result(frame) # The standard function callback code path pushes the FunctionCallResultFrame from the LLM @@ -217,11 +340,28 @@ class AWSNovaSonicAssistantContextAggregator(OpenAIAssistantContextAggregator): @dataclass class AWSNovaSonicContextAggregatorPair: + """Pair of user and assistant context aggregators for AWS Nova Sonic. + + Parameters: + _user: The user context aggregator. + _assistant: The assistant context aggregator. + """ + _user: AWSNovaSonicUserContextAggregator _assistant: AWSNovaSonicAssistantContextAggregator def user(self) -> AWSNovaSonicUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ return self._user def assistant(self) -> AWSNovaSonicAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ return self._assistant diff --git a/src/pipecat/services/aws_nova_sonic/frames.py b/src/pipecat/services/aws_nova_sonic/frames.py index 94d410f22..7d4feb2ae 100644 --- a/src/pipecat/services/aws_nova_sonic/frames.py +++ b/src/pipecat/services/aws_nova_sonic/frames.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Custom frames for AWS Nova Sonic LLM service.""" + from dataclasses import dataclass from pipecat.frames.frames import DataFrame, FunctionCallResultFrame @@ -11,4 +13,13 @@ from pipecat.frames.frames import DataFrame, FunctionCallResultFrame @dataclass class AWSNovaSonicFunctionCallResultFrame(DataFrame): + """Frame containing function call result for AWS Nova Sonic processing. + + This frame wraps a standard function call result frame to enable + AWS Nova Sonic-specific handling and context updates. + + Parameters: + result_frame: The underlying function call result frame. + """ + result_frame: FunctionCallResultFrame diff --git a/src/pipecat/services/azure/common.py b/src/pipecat/services/azure/common.py index 054463257..a6f1eeedd 100644 --- a/src/pipecat/services/azure/common.py +++ b/src/pipecat/services/azure/common.py @@ -4,14 +4,22 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import Optional +"""Language conversion utilities for Azure services.""" -from loguru import logger +from typing import Optional from pipecat.transcriptions.language import Language def language_to_azure_language(language: Language) -> Optional[str]: + """Convert a Language enum to Azure language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Azure language code, or None if not supported. + """ language_map = { # Afrikaans Language.AF: "af-ZA", diff --git a/src/pipecat/services/azure/image.py b/src/pipecat/services/azure/image.py index a1bae3af6..f07b86f83 100644 --- a/src/pipecat/services/azure/image.py +++ b/src/pipecat/services/azure/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure OpenAI image generation service implementation. + +This module provides integration with Azure's OpenAI image generation API +using REST endpoints for creating images from text prompts. +""" + import asyncio import io from typing import AsyncGenerator @@ -17,6 +23,13 @@ from pipecat.services.image_service import ImageGenService class AzureImageGenServiceREST(ImageGenService): + """Azure OpenAI REST-based image generation service. + + Provides image generation using Azure's OpenAI service via REST API. + Supports asynchronous image generation with polling for completion + and automatic image download and processing. + """ + def __init__( self, *, @@ -27,6 +40,16 @@ class AzureImageGenServiceREST(ImageGenService): aiohttp_session: aiohttp.ClientSession, api_version="2023-06-01-preview", ): + """Initialize the AzureImageGenServiceREST. + + Args: + image_size: Size specification for generated images (e.g., "1024x1024"). + api_key: Azure OpenAI API key for authentication. + endpoint: Azure OpenAI endpoint URL. + model: The image generation model to use. + aiohttp_session: Shared aiohttp session for HTTP requests. + api_version: Azure API version string. Defaults to "2023-06-01-preview". + """ super().__init__() self._api_key = api_key @@ -37,6 +60,15 @@ class AzureImageGenServiceREST(ImageGenService): self._aiohttp_session = aiohttp_session async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate an image from a text prompt using Azure OpenAI. + + Args: + prompt: The text prompt describing the desired image. + + Yields: + URLImageRawFrame containing the generated image data, or + ErrorFrame if generation fails. + """ url = f"{self._azure_endpoint}openai/images/generations:submit?api-version={self._api_version}" headers = {"api-key": self._api_key, "Content-Type": "application/json"} diff --git a/src/pipecat/services/azure/llm.py b/src/pipecat/services/azure/llm.py index 295a1f1c1..a4b93f2a4 100644 --- a/src/pipecat/services/azure/llm.py +++ b/src/pipecat/services/azure/llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure OpenAI service implementation for the Pipecat AI framework.""" + from loguru import logger from openai import AsyncAzureOpenAI @@ -15,13 +17,6 @@ class AzureLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Azure's OpenAI endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing Azure OpenAI - endpoint (str): The Azure endpoint URL - model (str): The model identifier to use - api_version (str, optional): Azure API version. Defaults to "2024-09-01-preview" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -33,6 +28,15 @@ class AzureLLMService(OpenAILLMService): api_version: str = "2024-09-01-preview", **kwargs, ): + """Initialize the Azure LLM service. + + Args: + api_key: The API key for accessing Azure OpenAI. + endpoint: The Azure endpoint URL. + model: The model identifier to use. + api_version: Azure API version. Defaults to "2024-09-01-preview". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ # Initialize variables before calling parent __init__() because that # will call create_client() and we need those values there. self._endpoint = endpoint @@ -40,7 +44,16 @@ class AzureLLMService(OpenAILLMService): super().__init__(api_key=api_key, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for Azure OpenAI endpoint.""" + """Create OpenAI-compatible client for Azure OpenAI endpoint. + + Args: + api_key: API key for authentication. Uses instance key if None. + base_url: Base URL for the client. Ignored for Azure implementation. + **kwargs: Additional keyword arguments. Ignored for Azure implementation. + + Returns: + AsyncAzureOpenAI: Configured Azure OpenAI client instance. + """ logger.debug(f"Creating Azure OpenAI client with endpoint {self._endpoint}") return AsyncAzureOpenAI( api_key=api_key, diff --git a/src/pipecat/services/azure/stt.py b/src/pipecat/services/azure/stt.py index abd8acbd3..415f91550 100644 --- a/src/pipecat/services/azure/stt.py +++ b/src/pipecat/services/azure/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure Speech-to-Text service implementation for Pipecat. + +This module provides speech-to-text functionality using Azure Cognitive Services +Speech SDK for real-time audio transcription. +""" + import asyncio from typing import AsyncGenerator, Optional @@ -40,6 +46,13 @@ except ModuleNotFoundError as e: class AzureSTTService(STTService): + """Azure Speech-to-Text service for real-time audio transcription. + + This service uses Azure Cognitive Services Speech SDK to convert speech + audio into text transcriptions. It supports continuous recognition and + provides real-time transcription results with timing information. + """ + def __init__( self, *, @@ -49,6 +62,15 @@ class AzureSTTService(STTService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the Azure STT service. + + Args: + api_key: Azure Cognitive Services subscription key. + region: Azure region for the Speech service (e.g., 'eastus'). + language: Language for speech recognition. Defaults to English (US). + sample_rate: Audio sample rate in Hz. If None, uses service default. + **kwargs: Additional arguments passed to parent STTService. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._speech_config = SpeechConfig( @@ -66,9 +88,25 @@ class AzureSTTService(STTService): } def can_generate_metrics(self) -> bool: + """Check if this service can generate performance metrics. + + Returns: + True as this service supports metrics generation. + """ return True async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text conversion. + + Feeds audio data to the Azure speech recognizer for processing. + Recognition results are handled asynchronously through callbacks. + + Args: + audio: Raw audio bytes to process. + + Yields: + None - actual transcription frames are pushed via callbacks. + """ await self.start_processing_metrics() await self.start_ttfb_metrics() if self._audio_stream: @@ -76,6 +114,14 @@ class AzureSTTService(STTService): yield None async def start(self, frame: StartFrame): + """Start the speech recognition service. + + Initializes the Azure speech recognizer with audio stream configuration + and begins continuous speech recognition. + + Args: + frame: Frame indicating the start of processing. + """ await super().start(frame) if self._audio_stream: @@ -93,6 +139,13 @@ class AzureSTTService(STTService): self._speech_recognizer.start_continuous_recognition_async() async def stop(self, frame: EndFrame): + """Stop the speech recognition service. + + Cleanly shuts down the Azure speech recognizer and closes audio streams. + + Args: + frame: Frame indicating the end of processing. + """ await super().stop(frame) if self._speech_recognizer: @@ -102,6 +155,13 @@ class AzureSTTService(STTService): self._audio_stream.close() async def cancel(self, frame: CancelFrame): + """Cancel the speech recognition service. + + Immediately stops recognition and closes resources. + + Args: + frame: Frame indicating cancellation. + """ await super().cancel(frame) if self._speech_recognizer: diff --git a/src/pipecat/services/azure/tts.py b/src/pipecat/services/azure/tts.py index e8be685a6..07e706a9b 100644 --- a/src/pipecat/services/azure/tts.py +++ b/src/pipecat/services/azure/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure Cognitive Services Text-to-Speech service implementations.""" + import asyncio from typing import AsyncGenerator, Optional @@ -39,6 +41,15 @@ except ModuleNotFoundError as e: def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputFormat: + """Convert sample rate to Azure speech synthesis output format. + + Args: + sample_rate: Sample rate in Hz. + + Returns: + Corresponding Azure SpeechSynthesisOutputFormat enum value. + Defaults to Raw24Khz16BitMonoPcm if sample rate not found. + """ sample_rate_map = { 8000: SpeechSynthesisOutputFormat.Raw8Khz16BitMonoPcm, 16000: SpeechSynthesisOutputFormat.Raw16Khz16BitMonoPcm, @@ -51,7 +62,26 @@ def sample_rate_to_output_format(sample_rate: int) -> SpeechSynthesisOutputForma class AzureBaseTTSService(TTSService): + """Base class for Azure Cognitive Services text-to-speech implementations. + + Provides common functionality for Azure TTS services including SSML + construction, voice configuration, and parameter management. + """ + class InputParams(BaseModel): + """Input parameters for Azure TTS voice configuration. + + Parameters: + emphasis: Emphasis level for speech ("strong", "moderate", "reduced"). + language: Language for synthesis. Defaults to English (US). + pitch: Voice pitch adjustment (e.g., "+10%", "-5Hz", "high"). + rate: Speech rate multiplier. Defaults to "1.05". + role: Voice role for expression (e.g., "YoungAdultFemale"). + style: Speaking style (e.g., "cheerful", "sad", "excited"). + style_degree: Intensity of the speaking style (0.01 to 2.0). + volume: Volume level (e.g., "+20%", "loud", "x-soft"). + """ + emphasis: Optional[str] = None language: Optional[Language] = Language.EN_US pitch: Optional[str] = None @@ -71,6 +101,16 @@ class AzureBaseTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Azure TTS service with configuration parameters. + + Args: + api_key: Azure Cognitive Services subscription key. + region: Azure region identifier (e.g., "eastus", "westus2"). + voice: Voice name to use for synthesis. Defaults to "en-US-SaraNeural". + sample_rate: Audio sample rate in Hz. If None, uses service default. + params: Voice and synthesis parameters configuration. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or AzureBaseTTSService.InputParams() @@ -94,9 +134,22 @@ class AzureBaseTTSService(TTSService): self._speech_synthesizer = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Azure TTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Azure language format. + + Args: + language: The language to convert. + + Returns: + The Azure-specific language code, or None if not supported. + """ return language_to_azure_language(language) def _construct_ssml(self, text: str) -> str: @@ -146,13 +199,30 @@ class AzureBaseTTSService(TTSService): class AzureTTSService(AzureBaseTTSService): + """Azure Cognitive Services streaming TTS service. + + Provides real-time text-to-speech synthesis using Azure's WebSocket-based + streaming API. Audio chunks are streamed as they become available for + lower latency playback. + """ + def __init__(self, **kwargs): + """Initialize the Azure streaming TTS service. + + Args: + **kwargs: All arguments passed to AzureBaseTTSService parent class. + """ super().__init__(**kwargs) self._speech_config = None self._speech_synthesizer = None self._audio_queue = asyncio.Queue() async def start(self, frame: StartFrame): + """Start the Azure TTS service and initialize speech synthesizer. + + Args: + frame: Start frame containing initialization parameters. + """ await super().start(frame) if self._speech_config: @@ -183,24 +253,33 @@ class AzureTTSService(AzureBaseTTSService): self._speech_synthesizer.synthesis_canceled.connect(self._handle_canceled) def _handle_synthesizing(self, evt): - """Handle audio chunks as they arrive""" + """Handle audio chunks as they arriv.""" if evt.result and evt.result.audio_data: self._audio_queue.put_nowait(evt.result.audio_data) def _handle_completed(self, evt): - """Handle synthesis completion""" + """Handle synthesis completion.""" self._audio_queue.put_nowait(None) # Signal completion def _handle_canceled(self, evt): - """Handle synthesis cancellation""" + """Handle synthesis cancellation.""" logger.error(f"Speech synthesis canceled: {evt.result.cancellation_details.reason}") self._audio_queue.put_nowait(None) async def flush_audio(self): + """Flush any pending audio data.""" logger.trace(f"{self}: flushing audio") @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Azure's streaming synthesis. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing synthesized speech data. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -244,12 +323,29 @@ class AzureTTSService(AzureBaseTTSService): class AzureHttpTTSService(AzureBaseTTSService): + """Azure Cognitive Services HTTP-based TTS service. + + Provides text-to-speech synthesis using Azure's HTTP API for simpler, + non-streaming synthesis. Suitable for use cases where streaming is not + required and simpler integration is preferred. + """ + def __init__(self, **kwargs): + """Initialize the Azure HTTP TTS service. + + Args: + **kwargs: All arguments passed to AzureBaseTTSService parent class. + """ super().__init__(**kwargs) self._speech_config = None self._speech_synthesizer = None async def start(self, frame: StartFrame): + """Start the Azure HTTP TTS service and initialize speech synthesizer. + + Args: + frame: Start frame containing initialization parameters. + """ await super().start(frame) if self._speech_config: @@ -269,6 +365,14 @@ class AzureHttpTTSService(AzureBaseTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Azure's HTTP synthesis API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the complete synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") await self.start_ttfb_metrics() diff --git a/src/pipecat/services/cartesia/stt.py b/src/pipecat/services/cartesia/stt.py index 104e4b2c5..5ca9cce0f 100644 --- a/src/pipecat/services/cartesia/stt.py +++ b/src/pipecat/services/cartesia/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Cartesia Speech-to-Text service implementation. + +This module provides a WebSocket-based STT service that integrates with +the Cartesia Live transcription API for real-time speech recognition. +""" + import asyncio import json import urllib.parse @@ -30,6 +36,12 @@ from pipecat.utils.tracing.service_decorators import traced_stt class CartesiaLiveOptions: + """Configuration options for Cartesia Live STT service. + + Manages transcription parameters including model selection, language, + audio encoding format, and sample rate settings. + """ + def __init__( self, *, @@ -39,6 +51,15 @@ class CartesiaLiveOptions: sample_rate: int = 16000, **kwargs, ): + """Initialize CartesiaLiveOptions with default or provided parameters. + + Args: + model: The transcription model to use. Defaults to "ink-whisper". + language: Target language for transcription. Defaults to English. + encoding: Audio encoding format. Defaults to "pcm_s16le". + sample_rate: Audio sample rate in Hz. Defaults to 16000. + **kwargs: Additional parameters for the transcription service. + """ self.model = model self.language = language self.encoding = encoding @@ -46,6 +67,11 @@ class CartesiaLiveOptions: self.additional_params = kwargs def to_dict(self): + """Convert options to dictionary format. + + Returns: + Dictionary containing all configuration parameters. + """ params = { "model": self.model, "language": self.language if isinstance(self.language, str) else self.language.value, @@ -56,19 +82,48 @@ class CartesiaLiveOptions: return params def items(self): + """Get configuration items as key-value pairs. + + Returns: + Iterator of (key, value) tuples for all configuration parameters. + """ return self.to_dict().items() def get(self, key, default=None): + """Get a configuration value by key. + + Args: + key: The configuration parameter name to retrieve. + default: Default value if key is not found. + + Returns: + The configuration value or default if not found. + """ if hasattr(self, key): return getattr(self, key) return self.additional_params.get(key, default) @classmethod def from_json(cls, json_str: str) -> "CartesiaLiveOptions": + """Create options from JSON string. + + Args: + json_str: JSON string containing configuration parameters. + + Returns: + New CartesiaLiveOptions instance with parsed parameters. + """ return cls(**json.loads(json_str)) class CartesiaSTTService(STTService): + """Speech-to-text service using Cartesia Live API. + + Provides real-time speech transcription through WebSocket connection + to Cartesia's Live transcription service. Supports both interim and + final transcriptions with configurable models and languages. + """ + def __init__( self, *, @@ -78,6 +133,15 @@ class CartesiaSTTService(STTService): live_options: Optional[CartesiaLiveOptions] = None, **kwargs, ): + """Initialize CartesiaSTTService with API key and options. + + Args: + api_key: Authentication key for Cartesia API. + base_url: Custom API endpoint URL. If empty, uses default. + sample_rate: Audio sample rate in Hz. Defaults to 16000. + live_options: Configuration options for transcription service. + **kwargs: Additional arguments passed to parent STTService. + """ sample_rate = sample_rate or (live_options.sample_rate if live_options else None) super().__init__(sample_rate=sample_rate, **kwargs) @@ -108,21 +172,49 @@ class CartesiaSTTService(STTService): self._receiver_task = None def can_generate_metrics(self) -> bool: + """Check if the service can generate processing metrics. + + Returns: + True, indicating metrics are supported. + """ return True async def start(self, frame: StartFrame): + """Start the STT service and establish connection. + + Args: + frame: Frame indicating service should start. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the STT service and close connection. + + Args: + frame: Frame indicating service should stop. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the STT service and close connection. + + Args: + frame: Frame indicating service should be cancelled. + """ await super().cancel(frame) await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None - transcription results are handled via WebSocket responses. + """ # If the connection is closed, due to timeout, we need to reconnect when the user starts speaking again if not self._connection or self._connection.closed: await self._connect() @@ -225,10 +317,17 @@ class CartesiaSTTService(STTService): self._connection = None async def start_metrics(self): + """Start performance metrics collection for transcription processing.""" await self.start_ttfb_metrics() await self.start_processing_metrics() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle speech events. + + Args: + frame: The frame to process. + direction: Direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): diff --git a/src/pipecat/services/cartesia/tts.py b/src/pipecat/services/cartesia/tts.py index 22493f229..68cab0600 100644 --- a/src/pipecat/services/cartesia/tts.py +++ b/src/pipecat/services/cartesia/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Cartesia text-to-speech service implementations.""" + import base64 import json import uuid @@ -27,6 +29,7 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.tts_service import AudioContextWordTTSService, TTSService from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.text.base_text_aggregator import BaseTextAggregator from pipecat.utils.text.skip_tags_aggregator import SkipTagsAggregator from pipecat.utils.tracing.service_decorators import traced_tts @@ -42,6 +45,14 @@ except ModuleNotFoundError as e: def language_to_cartesia_language(language: Language) -> Optional[str]: + """Convert a Language enum to Cartesia language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Cartesia language code, or None if not supported. + """ BASE_LANGUAGES = { Language.DE: "de", Language.EN: "en", @@ -74,7 +85,22 @@ def language_to_cartesia_language(language: Language) -> Optional[str]: class CartesiaTTSService(AudioContextWordTTSService): + """Cartesia TTS service with WebSocket streaming and word timestamps. + + Provides text-to-speech using Cartesia's streaming WebSocket API. + Supports word-level timestamps, audio context management, and various voice + customization options including speed and emotion controls. + """ + class InputParams(BaseModel): + """Input parameters for Cartesia TTS configuration. + + Parameters: + language: Language to use for synthesis. + speed: Voice speed control (string or float). + emotion: List of emotion controls (deprecated). + """ + language: Optional[Language] = Language.EN speed: Optional[Union[str, float]] = "" emotion: Optional[List[str]] = [] @@ -94,6 +120,21 @@ class CartesiaTTSService(AudioContextWordTTSService): text_aggregator: Optional[BaseTextAggregator] = None, **kwargs, ): + """Initialize the Cartesia TTS service. + + Args: + api_key: Cartesia API key for authentication. + voice_id: ID of the voice to use for synthesis. + cartesia_version: API version string for Cartesia service. + url: WebSocket URL for Cartesia TTS API. + model: TTS model to use (e.g., "sonic-2"). + sample_rate: Audio sample rate. If None, uses default. + encoding: Audio encoding format. + container: Audio container format. + params: Additional input parameters for voice customization. + text_aggregator: Custom text aggregator for processing input text. + **kwargs: Additional arguments passed to the parent service. + """ # Aggregating sentences still gives cleaner-sounding results and fewer # artifacts than streaming one word at a time. On average, waiting for a # full sentence should only "cost" us 15ms or so with GPT-4o or a Llama @@ -137,14 +178,32 @@ class CartesiaTTSService(AudioContextWordTTSService): self._receive_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Cartesia service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the TTS model. + + Args: + model: The model name to use for synthesis. + """ self._model_id = model await super().set_model(model) logger.info(f"Switching TTS model to: [{model}]") def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Cartesia language format. + + Args: + language: The language to convert. + + Returns: + The Cartesia-specific language code, or None if not supported. + """ return language_to_cartesia_language(language) def _build_msg( @@ -182,15 +241,30 @@ class CartesiaTTSService(AudioContextWordTTSService): return json.dumps(msg) async def start(self, frame: StartFrame): + """Start the Cartesia TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["output_format"]["sample_rate"] = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): + """Stop the Cartesia TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Stop the Cartesia TTS service. + + Args: + frame: The end frame. + """ await super().cancel(frame) await self._disconnect() @@ -247,6 +321,7 @@ class CartesiaTTSService(AudioContextWordTTSService): self._context_id = None async def flush_audio(self): + """Flush any pending audio and finalize the current context.""" if not self._context_id or not self._websocket: return logger.trace(f"{self}: flushing audio") @@ -255,7 +330,9 @@ class CartesiaTTSService(AudioContextWordTTSService): self._context_id = None async def _receive_messages(self): - async for message in self._get_websocket(): + async for message in WatchdogAsyncIterator( + self._get_websocket(), manager=self.task_manager + ): msg = json.loads(message) if not msg or not self.audio_context_available(msg["context_id"]): continue @@ -287,6 +364,14 @@ class CartesiaTTSService(AudioContextWordTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Cartesia's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -316,7 +401,22 @@ class CartesiaTTSService(AudioContextWordTTSService): class CartesiaHttpTTSService(TTSService): + """Cartesia HTTP-based TTS service. + + Provides text-to-speech using Cartesia's HTTP API for simpler, non-streaming + synthesis. Suitable for use cases where streaming is not required and simpler + integration is preferred. + """ + class InputParams(BaseModel): + """Input parameters for Cartesia HTTP TTS configuration. + + Parameters: + language: Language to use for synthesis. + speed: Voice speed control (string or float). + emotion: List of emotion controls (deprecated). + """ + language: Optional[Language] = Language.EN speed: Optional[Union[str, float]] = "" emotion: Optional[List[str]] = Field(default_factory=list) @@ -335,6 +435,20 @@ class CartesiaHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Cartesia HTTP TTS service. + + Args: + api_key: Cartesia API key for authentication. + voice_id: ID of the voice to use for synthesis. + model: TTS model to use (e.g., "sonic-2"). + base_url: Base URL for Cartesia HTTP API. + cartesia_version: API version string for Cartesia service. + sample_rate: Audio sample rate. If None, uses default. + encoding: Audio encoding format. + container: Audio container format. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or CartesiaHttpTTSService.InputParams() @@ -363,25 +477,61 @@ class CartesiaHttpTTSService(TTSService): ) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Cartesia HTTP service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Cartesia language format. + + Args: + language: The language to convert. + + Returns: + The Cartesia-specific language code, or None if not supported. + """ return language_to_cartesia_language(language) async def start(self, frame: StartFrame): + """Start the Cartesia HTTP TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["output_format"]["sample_rate"] = self.sample_rate async def stop(self, frame: EndFrame): + """Stop the Cartesia HTTP TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._client.close() async def cancel(self, frame: CancelFrame): + """Cancel the Cartesia HTTP TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._client.close() @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Cartesia's HTTP API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/cerebras/llm.py b/src/pipecat/services/cerebras/llm.py index 2217cc2f8..fb8e21a2b 100644 --- a/src/pipecat/services/cerebras/llm.py +++ b/src/pipecat/services/cerebras/llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Cerebras LLM service implementation using OpenAI-compatible interface.""" + from typing import List from loguru import logger @@ -19,12 +21,6 @@ class CerebrasLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Cerebras's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing Cerebras's API - base_url (str, optional): The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1" - model (str, optional): The model identifier to use. Defaults to "llama-3.3-70b" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -35,10 +31,27 @@ class CerebrasLLMService(OpenAILLMService): model: str = "llama-3.3-70b", **kwargs, ): + """Initialize the Cerebras LLM service. + + Args: + api_key: The API key for accessing Cerebras's API. + base_url: The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1". + model: The model identifier to use. Defaults to "llama-3.3-70b". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for Cerebras API endpoint.""" + """Create OpenAI-compatible client for Cerebras API endpoint. + + Args: + api_key: The API key for authentication. If None, uses instance key. + base_url: The base URL for the API. If None, uses instance URL. + **kwargs: Additional arguments passed to the client constructor. + + Returns: + An OpenAI-compatible client configured for Cerebras API. + """ logger.debug(f"Creating Cerebras client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) @@ -48,14 +61,14 @@ class CerebrasLLMService(OpenAILLMService): """Create a streaming chat completion using Cerebras's API. Args: - context (OpenAILLMContext): The context object containing tools configuration - and other settings for the chat completion. - messages (List[ChatCompletionMessageParam]): The list of messages comprising - the conversation history and current request. + context: The context object containing tools configuration + and other settings for the chat completion. + messages: The list of messages comprising + the conversation history and current request. Returns: - AsyncStream[ChatCompletionChunk]: A streaming response of chat completion - chunks that can be processed asynchronously. + A streaming response of chat completion + chunks that can be processed asynchronously. """ params = { "model": self.model_name, diff --git a/src/pipecat/services/deepgram/stt.py b/src/pipecat/services/deepgram/stt.py index 308ad1d1d..d897d8c92 100644 --- a/src/pipecat/services/deepgram/stt.py +++ b/src/pipecat/services/deepgram/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Deepgram speech-to-text service implementation.""" + from typing import AsyncGenerator, Dict, Optional from loguru import logger @@ -41,6 +43,13 @@ except ModuleNotFoundError as e: class DeepgramSTTService(STTService): + """Deepgram speech-to-text service. + + Provides real-time speech recognition using Deepgram's WebSocket API. + Supports configurable models, languages, VAD events, and various audio + processing options. + """ + def __init__( self, *, @@ -52,6 +61,17 @@ class DeepgramSTTService(STTService): addons: Optional[Dict] = None, **kwargs, ): + """Initialize the Deepgram STT service. + + Args: + api_key: Deepgram API key for authentication. + url: Deprecated. Use base_url instead. + base_url: Custom Deepgram API base URL. + sample_rate: Audio sample rate. If None, uses default or live_options value. + live_options: Deepgram LiveOptions for detailed configuration. + addons: Additional Deepgram features to enable. + **kwargs: Additional arguments passed to the parent STTService. + """ sample_rate = sample_rate or (live_options.sample_rate if live_options else None) super().__init__(sample_rate=sample_rate, **kwargs) @@ -108,12 +128,27 @@ class DeepgramSTTService(STTService): @property def vad_enabled(self): + """Check if Deepgram VAD events are enabled. + + Returns: + True if VAD events are enabled in the current settings. + """ return self._settings["vad_events"] def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Deepgram service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the Deepgram model and reconnect. + + Args: + model: The Deepgram model name to use. + """ await super().set_model(model) logger.info(f"Switching STT model to: [{model}]") self._settings["model"] = model @@ -121,25 +156,53 @@ class DeepgramSTTService(STTService): await self._connect() async def set_language(self, language: Language): + """Set the recognition language and reconnect. + + Args: + language: The language to use for speech recognition. + """ logger.info(f"Switching STT language to: [{language}]") self._settings["language"] = language await self._disconnect() await self._connect() async def start(self, frame: StartFrame): + """Start the Deepgram STT service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["sample_rate"] = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): + """Stop the Deepgram STT service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the Deepgram STT service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Send audio data to Deepgram for transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: None (transcription results come via WebSocket callbacks). + """ await self._connection.send(audio) yield None @@ -172,6 +235,7 @@ class DeepgramSTTService(STTService): await self._connection.finish() async def start_metrics(self): + """Start TTFB and processing metrics collection.""" await self.start_ttfb_metrics() await self.start_processing_metrics() @@ -235,6 +299,12 @@ class DeepgramSTTService(STTService): ) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with Deepgram-specific handling. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame) and not self.vad_enabled: diff --git a/src/pipecat/services/deepgram/tts.py b/src/pipecat/services/deepgram/tts.py index a684a340e..5819e4123 100644 --- a/src/pipecat/services/deepgram/tts.py +++ b/src/pipecat/services/deepgram/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Deepgram text-to-speech service implementation. + +This module provides integration with Deepgram's text-to-speech API +for generating speech from text using various voice models. +""" + from typing import AsyncGenerator, Optional from loguru import logger @@ -27,6 +33,13 @@ except ModuleNotFoundError as e: class DeepgramTTSService(TTSService): + """Deepgram text-to-speech service. + + Provides text-to-speech synthesis using Deepgram's streaming API. + Supports various voice models and audio encoding formats with + configurable sample rates and quality settings. + """ + def __init__( self, *, @@ -37,6 +50,16 @@ class DeepgramTTSService(TTSService): encoding: str = "linear16", **kwargs, ): + """Initialize the Deepgram TTS service. + + Args: + api_key: Deepgram API key for authentication. + voice: Voice model to use for synthesis. Defaults to "aura-2-helena-en". + base_url: Custom base URL for Deepgram API. Uses default if empty. + sample_rate: Audio sample rate in Hz. If None, uses service default. + encoding: Audio encoding format. Defaults to "linear16". + **kwargs: Additional arguments passed to parent TTSService class. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._settings = { @@ -48,10 +71,23 @@ class DeepgramTTSService(TTSService): self._deepgram_client = DeepgramClient(api_key, config=client_options) def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + True, as Deepgram TTS service supports metrics generation. + """ return True @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Deepgram's TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech, plus start/stop frames. + """ logger.debug(f"{self}: Generating TTS [{text}]") options = SpeakOptions( diff --git a/src/pipecat/services/deepseek/llm.py b/src/pipecat/services/deepseek/llm.py index 7bed5d33b..55aaa341f 100644 --- a/src/pipecat/services/deepseek/llm.py +++ b/src/pipecat/services/deepseek/llm.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""DeepSeek LLM service implementation using OpenAI-compatible interface.""" from typing import List @@ -20,12 +21,6 @@ class DeepSeekLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to DeepSeek's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing DeepSeek's API - base_url (str, optional): The base URL for DeepSeek API. Defaults to "https://api.deepseek.com/v1" - model (str, optional): The model identifier to use. Defaults to "deepseek-chat" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -36,27 +31,44 @@ class DeepSeekLLMService(OpenAILLMService): model: str = "deepseek-chat", **kwargs, ): + """Initialize the DeepSeek LLM service. + + Args: + api_key: The API key for accessing DeepSeek's API. + base_url: The base URL for DeepSeek API. Defaults to "https://api.deepseek.com/v1". + model: The model identifier to use. Defaults to "deepseek-chat". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for DeepSeek API endpoint.""" + """Create OpenAI-compatible client for DeepSeek API endpoint. + + Args: + api_key: The API key for authentication. If None, uses instance default. + base_url: The base URL for the API. If None, uses instance default. + **kwargs: Additional keyword arguments for client configuration. + + Returns: + An OpenAI-compatible client configured for DeepSeek's API. + """ logger.debug(f"Creating DeepSeek client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) async def get_chat_completions( self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] ) -> AsyncStream[ChatCompletionChunk]: - """Create a streaming chat completion using Cerebras's API. + """Create a streaming chat completion using DeepSeek's API. Args: - context (OpenAILLMContext): The context object containing tools configuration - and other settings for the chat completion. - messages (List[ChatCompletionMessageParam]): The list of messages comprising - the conversation history and current request. + context: The context object containing tools configuration + and other settings for the chat completion. + messages: The list of messages comprising the conversation + history and current request. Returns: - AsyncStream[ChatCompletionChunk]: A streaming response of chat completion - chunks that can be processed asynchronously. + A streaming response of chat completion chunks that can be + processed asynchronously. """ params = { "model": self.model_name, diff --git a/src/pipecat/services/elevenlabs/tts.py b/src/pipecat/services/elevenlabs/tts.py index ccd9b5b3f..7153c39c5 100644 --- a/src/pipecat/services/elevenlabs/tts.py +++ b/src/pipecat/services/elevenlabs/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""ElevenLabs text-to-speech service implementations. + +This module provides WebSocket and HTTP-based TTS services using ElevenLabs API +with support for streaming audio, word timestamps, and voice customization. +""" + import asyncio import base64 import json @@ -32,6 +38,7 @@ from pipecat.services.tts_service import ( WordTTSService, ) from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.tracing.service_decorators import traced_tts # See .env.example for ElevenLabs configuration needed @@ -56,6 +63,14 @@ ELEVENLABS_MULTILINGUAL_MODELS = { def language_to_elevenlabs_language(language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding ElevenLabs language code, or None if not supported. + """ BASE_LANGUAGES = { Language.AR: "ar", Language.BG: "bg", @@ -105,6 +120,14 @@ def language_to_elevenlabs_language(language: Language) -> Optional[str]: def output_format_from_sample_rate(sample_rate: int) -> str: + """Get the appropriate output format string for a given sample rate. + + Args: + sample_rate: The audio sample rate in Hz. + + Returns: + The ElevenLabs output format string. + """ match sample_rate: case 8000: return "pcm_8000" @@ -128,10 +151,10 @@ def build_elevenlabs_voice_settings( """Build voice settings dictionary for ElevenLabs based on provided settings. Args: - settings: Dictionary containing voice settings parameters + settings: Dictionary containing voice settings parameters. Returns: - Dictionary of voice settings or None if no valid settings are provided + Dictionary of voice settings or None if no valid settings are provided. """ voice_setting_keys = ["stability", "similarity_boost", "style", "use_speaker_boost", "speed"] @@ -146,6 +169,15 @@ def build_elevenlabs_voice_settings( def calculate_word_times( alignment_info: Mapping[str, Any], cumulative_time: float ) -> List[Tuple[str, float]]: + """Calculate word timestamps from character alignment information. + + Args: + alignment_info: Character alignment data from ElevenLabs API. + cumulative_time: Base time offset for this chunk. + + Returns: + List of (word, timestamp) tuples. + """ zipped_times = list(zip(alignment_info["chars"], alignment_info["charStartTimesMs"])) words = "".join(alignment_info["chars"]).split(" ") @@ -165,7 +197,28 @@ def calculate_word_times( class ElevenLabsTTSService(AudioContextWordTTSService): + """ElevenLabs WebSocket-based TTS service with word timestamps. + + Provides real-time text-to-speech using ElevenLabs' WebSocket streaming API. + Supports word-level timestamps, audio context management, and various voice + customization options including stability, similarity boost, and speed controls. + """ + class InputParams(BaseModel): + """Input parameters for ElevenLabs TTS configuration. + + Parameters: + language: Language to use for synthesis. + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.25 to 4.0). + auto_mode: Whether to enable automatic mode optimization. + enable_ssml_parsing: Whether to parse SSML tags in text. + enable_logging: Whether to enable ElevenLabs logging. + """ + language: Optional[Language] = None stability: Optional[float] = None similarity_boost: Optional[float] = None @@ -187,6 +240,17 @@ class ElevenLabsTTSService(AudioContextWordTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the ElevenLabs TTS service. + + Args: + api_key: ElevenLabs API key for authentication. + voice_id: ID of the voice to use for synthesis. + model: TTS model to use (e.g., "eleven_flash_v2_5"). + url: WebSocket URL for ElevenLabs TTS API. + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent service. + """ # Aggregating sentences still gives cleaner-sounding results and fewer # artifacts than streaming one word at a time. On average, waiting for a # full sentence should only "cost" us 15ms or so with GPT-4o or a Llama @@ -243,21 +307,40 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._keepalive_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as ElevenLabs service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to ElevenLabs language format. + + Args: + language: The language to convert. + + Returns: + The ElevenLabs-specific language code, or None if not supported. + """ return language_to_elevenlabs_language(language) def _set_voice_settings(self): return build_elevenlabs_voice_settings(self._settings) async def set_model(self, model: str): + """Set the TTS model and reconnect. + + Args: + model: The model name to use for synthesis. + """ await super().set_model(model) logger.info(f"Switching TTS model to: [{model}]") await self._disconnect() await self._connect() async def _update_settings(self, settings: Mapping[str, Any]): + """Update service settings and reconnect if voice changed.""" prev_voice = self._voice_id await super()._update_settings(settings) if not prev_voice == self._voice_id: @@ -266,19 +349,35 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._connect() async def start(self, frame: StartFrame): + """Start the ElevenLabs TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._output_format = output_format_from_sample_rate(self.sample_rate) await self._connect() async def stop(self, frame: EndFrame): + """Stop the ElevenLabs TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the ElevenLabs TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def flush_audio(self): + """Flush any pending audio and finalize the current context.""" if not self._context_id or not self._websocket: return logger.trace(f"{self}: flushing audio") @@ -286,6 +385,12 @@ class ElevenLabsTTSService(AudioContextWordTTSService): await self._websocket.send(json.dumps(msg)) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False @@ -373,6 +478,7 @@ class ElevenLabsTTSService(AudioContextWordTTSService): raise Exception("Websocket not connected") async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + """Handle interruption by closing the current context.""" await super()._handle_interruption(frame, direction) # Close the current context when interrupted without closing the websocket @@ -394,7 +500,10 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._started = False async def _receive_messages(self): - async for message in self._get_websocket(): + """Handle incoming WebSocket messages from ElevenLabs.""" + async for message in WatchdogAsyncIterator( + self._get_websocket(), manager=self.task_manager + ): msg = json.loads(message) received_ctx_id = msg.get("contextId") @@ -425,8 +534,11 @@ class ElevenLabsTTSService(AudioContextWordTTSService): self._cumulative_time = word_times[-1][1] async def _keepalive_task_handler(self): + """Send periodic keepalive messages to maintain WebSocket connection.""" + KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3 while True: - await asyncio.sleep(10) + self.reset_watchdog() + await asyncio.sleep(KEEPALIVE_SLEEP) try: if self._websocket and self._websocket.open: if self._context_id: @@ -448,12 +560,21 @@ class ElevenLabsTTSService(AudioContextWordTTSService): break async def _send_text(self, text: str): + """Send text to the WebSocket for synthesis.""" if self._websocket and self._context_id: msg = {"text": text, "context_id": self._context_id} await self._websocket.send(json.dumps(msg)) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using ElevenLabs' streaming WebSocket API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -492,19 +613,26 @@ class ElevenLabsTTSService(AudioContextWordTTSService): class ElevenLabsHttpTTSService(WordTTSService): - """ElevenLabs Text-to-Speech service using HTTP streaming with word timestamps. + """ElevenLabs HTTP-based TTS service with word timestamps. - Args: - api_key: ElevenLabs API key - voice_id: ID of the voice to use - aiohttp_session: aiohttp ClientSession - model: Model ID (default: "eleven_flash_v2_5" for low latency) - base_url: API base URL - sample_rate: Output sample rate - params: Additional parameters for voice configuration + Provides text-to-speech using ElevenLabs' HTTP streaming API for simpler, + non-WebSocket integration. Suitable for use cases where streaming WebSocket + connection is not required or desired. """ class InputParams(BaseModel): + """Input parameters for ElevenLabs HTTP TTS configuration. + + Parameters: + language: Language to use for synthesis. + optimize_streaming_latency: Latency optimization level (0-4). + stability: Voice stability control (0.0 to 1.0). + similarity_boost: Similarity boost control (0.0 to 1.0). + style: Style control for voice expression (0.0 to 1.0). + use_speaker_boost: Whether to use speaker boost enhancement. + speed: Voice speed control (0.25 to 4.0). + """ + language: Optional[Language] = None optimize_streaming_latency: Optional[int] = None stability: Optional[float] = None @@ -525,6 +653,18 @@ class ElevenLabsHttpTTSService(WordTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the ElevenLabs HTTP TTS service. + + Args: + api_key: ElevenLabs API key for authentication. + voice_id: ID of the voice to use for synthesis. + aiohttp_session: aiohttp ClientSession for HTTP requests. + model: TTS model to use (e.g., "eleven_flash_v2_5"). + base_url: Base URL for ElevenLabs HTTP API. + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent service. + """ super().__init__( aggregate_sentences=True, push_text_frames=False, @@ -564,11 +704,22 @@ class ElevenLabsHttpTTSService(WordTTSService): self._previous_text = "" def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert pipecat Language to ElevenLabs language code.""" + """Convert pipecat Language to ElevenLabs language code. + + Args: + language: The language to convert. + + Returns: + The ElevenLabs-specific language code, or None if not supported. + """ return language_to_elevenlabs_language(language) def can_generate_metrics(self) -> bool: - """Indicate that this service can generate usage metrics.""" + """Check if this service can generate processing metrics. + + Returns: + True, as ElevenLabs HTTP service supports metrics generation. + """ return True def _set_voice_settings(self): @@ -582,12 +733,22 @@ class ElevenLabsHttpTTSService(WordTTSService): logger.debug(f"{self}: Reset internal state") async def start(self, frame: StartFrame): - """Initialize the service upon receiving a StartFrame.""" + """Start the ElevenLabs HTTP TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._output_format = output_format_from_sample_rate(self.sample_rate) self._reset_state() async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame and handle state changes. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (StartInterruptionFrame, TTSStoppedFrame)): # Reset timing on interruption or stop @@ -614,10 +775,10 @@ class ElevenLabsHttpTTSService(WordTTSService): [("Hello", 0.1), ("world", 0.5)] Args: - alignment_info: Character timing data from ElevenLabs + alignment_info: Character timing data from ElevenLabs. Returns: - List of (word, timestamp) pairs + List of (word, timestamp) pairs. """ chars = alignment_info.get("characters", []) char_start_times = alignment_info.get("character_start_times_seconds", []) @@ -668,10 +829,10 @@ class ElevenLabsHttpTTSService(WordTTSService): Includes previous text as context for better prosody continuity. Args: - text: Text to convert to speech + text: Text to convert to speech. Yields: - Audio and control frames + Frame: Audio and control frames containing the synthesized speech. """ logger.debug(f"{self}: Generating TTS [{text}]") diff --git a/src/pipecat/services/fal/image.py b/src/pipecat/services/fal/image.py index 78439486e..c110ec3be 100644 --- a/src/pipecat/services/fal/image.py +++ b/src/pipecat/services/fal/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Fal's image generation service implementation. + +This module provides integration with Fal's image generation API +for creating images from text prompts using various AI models. +""" + import asyncio import io import os @@ -26,7 +32,25 @@ except ModuleNotFoundError as e: class FalImageGenService(ImageGenService): + """Fal's image generation service. + + Provides text-to-image generation using Fal.ai's API with configurable + parameters for image quality, safety, and format options. + """ + class InputParams(BaseModel): + """Input parameters for Fal.ai image generation. + + Parameters: + seed: Random seed for reproducible generation. If None, uses random seed. + num_inference_steps: Number of inference steps for generation. Defaults to 8. + num_images: Number of images to generate. Defaults to 1. + image_size: Image dimensions as string preset or dict with width/height. Defaults to "square_hd". + expand_prompt: Whether to automatically expand/enhance the prompt. Defaults to False. + enable_safety_checker: Whether to enable content safety filtering. Defaults to True. + format: Output image format. Defaults to "png". + """ + seed: Optional[int] = None num_inference_steps: int = 8 num_images: int = 1 @@ -44,6 +68,15 @@ class FalImageGenService(ImageGenService): key: Optional[str] = None, **kwargs, ): + """Initialize the FalImageGenService. + + Args: + params: Input parameters for image generation configuration. + aiohttp_session: HTTP client session for downloading generated images. + model: The Fal.ai model to use for generation. Defaults to "fal-ai/fast-sdxl". + key: Optional API key for Fal.ai. If provided, sets FAL_KEY environment variable. + **kwargs: Additional arguments passed to parent ImageGenService. + """ super().__init__(**kwargs) self.set_model_name(model) self._params = params @@ -52,6 +85,16 @@ class FalImageGenService(ImageGenService): os.environ["FAL_KEY"] = key async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate an image from a text prompt. + + Args: + prompt: The text prompt to generate an image from. + + Yields: + URLImageRawFrame: Frame containing the generated image data and metadata. + ErrorFrame: If image generation fails. + """ + def load_image_bytes(encoded_image: bytes): buffer = io.BytesIO(encoded_image) image = Image.open(buffer) diff --git a/src/pipecat/services/fal/stt.py b/src/pipecat/services/fal/stt.py index 1e26d9958..3485a7de1 100644 --- a/src/pipecat/services/fal/stt.py +++ b/src/pipecat/services/fal/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Fal speech-to-text service implementation. + +This module provides integration with Fal's Wizper API for speech-to-text +transcription using segmented audio processing. +""" + import os from typing import AsyncGenerator, Optional @@ -27,7 +33,14 @@ except ModuleNotFoundError as e: def language_to_fal_language(language: Language) -> Optional[str]: - """Language support for Fal's Wizper API.""" + """Convert a Language enum to Fal's Wizper language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Fal Wizper language code, or None if not supported. + """ BASE_LANGUAGES = { Language.AF: "af", Language.AM: "am", @@ -145,18 +158,12 @@ class FalSTTService(SegmentedSTTService): This service uses Fal's Wizper API to perform speech-to-text transcription on audio segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. - - Args: - api_key: Fal API key. If not provided, will check FAL_KEY environment variable. - sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. - params: Configuration parameters for the Wizper API. - **kwargs: Additional arguments passed to SegmentedSTTService. """ class InputParams(BaseModel): """Configuration parameters for Fal's Wizper API. - Attributes: + Parameters: language: Language of the audio input. Defaults to English. task: Task to perform ('transcribe' or 'translate'). Defaults to 'transcribe'. chunk_level: Level of chunking ('segment'). Defaults to 'segment'. @@ -176,6 +183,14 @@ class FalSTTService(SegmentedSTTService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the FalSTTService with API key and parameters. + + Args: + api_key: Fal API key. If not provided, will check FAL_KEY environment variable. + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate. + params: Configuration parameters for the Wizper API. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ super().__init__( sample_rate=sample_rate, **kwargs, @@ -201,16 +216,39 @@ class FalSTTService(SegmentedSTTService): } def can_generate_metrics(self) -> bool: + """Check if the service can generate processing metrics. + + Returns: + True, as Fal STT service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Fal's service-specific language code. + + Args: + language: The language to convert. + + Returns: + The Fal-specific language code, or None if not supported. + """ return language_to_fal_language(language) async def set_language(self, language: Language): + """Set the transcription language. + + Args: + language: The language to use for speech-to-text transcription. + """ logger.info(f"Switching STT language to: [{language}]") self._settings["language"] = self.language_to_service_language(language) async def set_model(self, model: str): + """Set the STT model. + + Args: + model: The model name to use for transcription. + """ await super().set_model(model) logger.info(f"Switching STT model to: [{model}]") @@ -229,7 +267,7 @@ class FalSTTService(SegmentedSTTService): audio: Raw audio bytes in WAV format (already converted by base class). Yields: - Frame: TranscriptionFrame containing the transcribed text. + Frame: TranscriptionFrame containing the transcribed text, or ErrorFrame on failure. Note: The audio is already in WAV format from the SegmentedSTTService. diff --git a/src/pipecat/services/fireworks/llm.py b/src/pipecat/services/fireworks/llm.py index d4003f86f..9edd6215c 100644 --- a/src/pipecat/services/fireworks/llm.py +++ b/src/pipecat/services/fireworks/llm.py @@ -4,6 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Fireworks AI service implementation using OpenAI-compatible interface.""" from typing import List @@ -19,12 +20,6 @@ class FireworksLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Fireworks' API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing Fireworks AI - model (str, optional): The model identifier to use. Defaults to "accounts/fireworks/models/firefunction-v2" - base_url (str, optional): The base URL for Fireworks API. Defaults to "https://api.fireworks.ai/inference/v1" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -35,10 +30,27 @@ class FireworksLLMService(OpenAILLMService): base_url: str = "https://api.fireworks.ai/inference/v1", **kwargs, ): + """Initialize the Fireworks LLM service. + + Args: + api_key: The API key for accessing Fireworks AI. + model: The model identifier to use. Defaults to "accounts/fireworks/models/firefunction-v2". + base_url: The base URL for Fireworks API. Defaults to "https://api.fireworks.ai/inference/v1". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for Fireworks API endpoint.""" + """Create OpenAI-compatible client for Fireworks API endpoint. + + Args: + api_key: API key for authentication. If None, uses instance default. + base_url: Base URL for the API. If None, uses instance default. + **kwargs: Additional arguments passed to the client constructor. + + Returns: + Configured OpenAI client instance for Fireworks API. + """ logger.debug(f"Creating Fireworks client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) @@ -47,7 +59,15 @@ class FireworksLLMService(OpenAILLMService): ): """Get chat completions from Fireworks API. - Removes OpenAI-specific parameters not supported by Fireworks. + Removes OpenAI-specific parameters not supported by Fireworks and + configures the request with Fireworks-compatible settings. + + Args: + context: The OpenAI LLM context containing tools and settings. + messages: List of chat completion message parameters. + + Returns: + Async generator yielding chat completion chunks from Fireworks API. """ params = { "model": self.model_name, diff --git a/src/pipecat/services/fish/tts.py b/src/pipecat/services/fish/tts.py index 84e285aee..7f2b85bdb 100644 --- a/src/pipecat/services/fish/tts.py +++ b/src/pipecat/services/fish/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Fish Audio text-to-speech service implementation. + +This module provides integration with Fish Audio's real-time TTS WebSocket API +for streaming text-to-speech synthesis with customizable voice parameters. +""" + import uuid from typing import AsyncGenerator, Literal, Optional @@ -39,7 +45,23 @@ FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] class FishAudioTTSService(InterruptibleTTSService): + """Fish Audio text-to-speech service with WebSocket streaming. + + Provides real-time text-to-speech synthesis using Fish Audio's WebSocket API. + Supports various audio formats, customizable prosody controls, and streaming + audio generation with interruption handling. + """ + class InputParams(BaseModel): + """Input parameters for Fish Audio TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + latency: Latency mode ("normal" or "balanced"). Defaults to "normal". + prosody_speed: Speech speed multiplier (0.5-2.0). Defaults to 1.0. + prosody_volume: Volume adjustment in dB. Defaults to 0. + """ + language: Optional[Language] = Language.EN latency: Optional[str] = "normal" # "normal" or "balanced" prosody_speed: Optional[float] = 1.0 # Speech speed (0.5-2.0) @@ -55,6 +77,16 @@ class FishAudioTTSService(InterruptibleTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Fish Audio TTS service. + + Args: + api_key: Fish Audio API key for authentication. + model: Reference ID of the voice model to use for synthesis. + output_format: Audio output format. Defaults to "pcm". + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to the parent service. + """ super().__init__( push_stop_frames=True, pause_frame_processing=True, @@ -85,23 +117,48 @@ class FishAudioTTSService(InterruptibleTTSService): self.set_model_name(model) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Fish Audio service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the TTS model (reference ID). + + Args: + model: The reference ID of the voice model to use. + """ self._settings["reference_id"] = model await super().set_model(model) logger.info(f"Switching TTS model to: [{model}]") async def start(self, frame: StartFrame): + """Start the Fish Audio TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["sample_rate"] = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): + """Stop the Fish Audio TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the Fish Audio TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() @@ -191,6 +248,14 @@ class FishAudioTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Fish Audio's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames and control frames for the synthesized speech. + """ logger.debug(f"{self}: Generating Fish TTS: [{text}]") try: if not self._websocket or self._websocket.closed: diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index 037a2e5f4..dced95d6c 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -3,7 +3,8 @@ # # SPDX-License-Identifier: BSD 2-Clause License # -# + +"""Event models and utilities for Google Gemini Multimodal Live API.""" import base64 import io @@ -23,11 +24,25 @@ from pipecat.frames.frames import ImageRawFrame class MediaChunk(BaseModel): + """Represents a chunk of media data for transmission. + + Parameters: + mimeType: MIME type of the media content. + data: Base64-encoded media data. + """ + mimeType: str data: str class ContentPart(BaseModel): + """Represents a part of content that can contain text or media. + + Parameters: + text: Text content. Defaults to None. + inlineData: Inline media data. Defaults to None. + """ + text: Optional[str] = Field(default=None, validate_default=False) inlineData: Optional[MediaChunk] = Field(default=None, validate_default=False) fileData: Optional['FileData'] = Field(default=None, validate_default=False) @@ -43,6 +58,13 @@ ContentPart.model_rebuild() # Rebuild model to resolve forward reference class Turn(BaseModel): + """Represents a conversational turn in the dialogue. + + Parameters: + role: The role of the speaker, either "user" or "model". Defaults to "user". + parts: List of content parts that make up the turn. + """ + role: Literal["user", "model"] = "user" parts: List[ContentPart] @@ -64,7 +86,15 @@ class EndSensitivity(str, Enum): class AutomaticActivityDetection(BaseModel): - """Configures automatic detection of activity.""" + """Configures automatic detection of voice activity. + + Parameters: + disabled: Whether automatic activity detection is disabled. Defaults to None. + start_of_speech_sensitivity: Sensitivity for detecting speech start. Defaults to None. + prefix_padding_ms: Padding before speech start in milliseconds. Defaults to None. + end_of_speech_sensitivity: Sensitivity for detecting speech end. Defaults to None. + silence_duration_ms: Duration of silence to detect speech end. Defaults to None. + """ disabled: Optional[bool] = None start_of_speech_sensitivity: Optional[StartSensitivity] = None @@ -74,25 +104,57 @@ class AutomaticActivityDetection(BaseModel): class RealtimeInputConfig(BaseModel): - """Configures the realtime input behavior.""" + """Configures the realtime input behavior. + + Parameters: + automatic_activity_detection: Voice activity detection configuration. Defaults to None. + """ automatic_activity_detection: Optional[AutomaticActivityDetection] = None class RealtimeInput(BaseModel): + """Contains realtime input media chunks. + + Parameters: + mediaChunks: List of media chunks for realtime processing. + """ + mediaChunks: List[MediaChunk] class ClientContent(BaseModel): + """Content sent from client to the Gemini Live API. + + Parameters: + turns: List of conversation turns. Defaults to None. + turnComplete: Whether the client's turn is complete. Defaults to False. + """ + turns: Optional[List[Turn]] = None turnComplete: bool = False class AudioInputMessage(BaseModel): + """Message containing audio input data. + + Parameters: + realtimeInput: Realtime input containing audio chunks. + """ + realtimeInput: RealtimeInput @classmethod def from_raw_audio(cls, raw_audio: bytes, sample_rate: int) -> "AudioInputMessage": + """Create an audio input message from raw audio data. + + Args: + raw_audio: Raw audio bytes. + sample_rate: Audio sample rate in Hz. + + Returns: + AudioInputMessage instance with encoded audio data. + """ data = base64.b64encode(raw_audio).decode("utf-8") return cls( realtimeInput=RealtimeInput( @@ -102,10 +164,24 @@ class AudioInputMessage(BaseModel): class VideoInputMessage(BaseModel): + """Message containing video/image input data. + + Parameters: + realtimeInput: Realtime input containing video/image chunks. + """ + realtimeInput: RealtimeInput @classmethod def from_image_frame(cls, frame: ImageRawFrame) -> "VideoInputMessage": + """Create a video input message from an image frame. + + Args: + frame: Image frame to encode. + + Returns: + VideoInputMessage instance with encoded image data. + """ buffer = io.BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") data = base64.b64encode(buffer.getvalue()).decode("utf-8") @@ -115,18 +191,44 @@ class VideoInputMessage(BaseModel): class ClientContentMessage(BaseModel): + """Message containing client content for the API. + + Parameters: + clientContent: The client content to send. + """ + clientContent: ClientContent class SystemInstruction(BaseModel): + """System instruction for the model. + + Parameters: + parts: List of content parts that make up the system instruction. + """ + parts: List[ContentPart] class AudioTranscriptionConfig(BaseModel): + """Configuration for audio transcription.""" + pass class Setup(BaseModel): + """Setup configuration for the Gemini Live session. + + Parameters: + model: Model identifier to use. + system_instruction: System instruction for the model. Defaults to None. + tools: List of available tools/functions. Defaults to None. + generation_config: Generation configuration parameters. Defaults to None. + input_audio_transcription: Input audio transcription config. Defaults to None. + output_audio_transcription: Output audio transcription config. Defaults to None. + realtime_input_config: Realtime input configuration. Defaults to None. + """ + model: str system_instruction: Optional[SystemInstruction] = None tools: Optional[List[dict]] = None @@ -137,6 +239,12 @@ class Setup(BaseModel): class Config(BaseModel): + """Configuration message for session setup. + + Parameters: + setup: Setup configuration for the session. + """ + setup: Setup @@ -189,36 +297,86 @@ class GroundingMetadata(BaseModel): class SetupComplete(BaseModel): + """Indicates that session setup is complete.""" + pass class InlineData(BaseModel): + """Inline data embedded in server responses. + + Parameters: + mimeType: MIME type of the data. + data: Base64-encoded data content. + """ + mimeType: str data: str class Part(BaseModel): + """Part of a server response containing data or text. + + Parameters: + inlineData: Inline binary data. Defaults to None. + text: Text content. Defaults to None. + """ + inlineData: Optional[InlineData] = None text: Optional[str] = None class ModelTurn(BaseModel): + """Represents a turn from the model in the conversation. + + Parameters: + parts: List of content parts in the model's response. + """ + parts: List[Part] class ServerContentInterrupted(BaseModel): + """Indicates server content was interrupted. + + Parameters: + interrupted: Whether the content was interrupted. + """ + interrupted: bool class ServerContentTurnComplete(BaseModel): + """Indicates the server's turn is complete. + + Parameters: + turnComplete: Whether the turn is complete. + """ + turnComplete: bool class BidiGenerateContentTranscription(BaseModel): + """Transcription data from bidirectional content generation. + + Parameters: + text: The transcribed text content. + """ + text: str class ServerContent(BaseModel): + """Content sent from server to client. + + Parameters: + modelTurn: Model's conversational turn. Defaults to None. + interrupted: Whether content was interrupted. Defaults to None. + turnComplete: Whether the turn is complete. Defaults to None. + inputTranscription: Transcription of input audio. Defaults to None. + outputTranscription: Transcription of output audio. Defaults to None. + """ + modelTurn: Optional[ModelTurn] = None interrupted: Optional[bool] = None turnComplete: Optional[bool] = None @@ -228,12 +386,26 @@ class ServerContent(BaseModel): class FunctionCall(BaseModel): + """Represents a function call from the model. + + Parameters: + id: Unique identifier for the function call. + name: Name of the function to call. + args: Arguments to pass to the function. + """ + id: str name: str args: dict class ToolCall(BaseModel): + """Contains one or more function calls. + + Parameters: + functionCalls: List of function calls to execute. + """ + functionCalls: List[FunctionCall] @@ -248,14 +420,32 @@ class Modality(str, Enum): class ModalityTokenCount(BaseModel): - """Token count for a specific modality.""" + """Token count for a specific modality. + + Parameters: + modality: The modality type. + tokenCount: Number of tokens for this modality. + """ modality: Modality tokenCount: int class UsageMetadata(BaseModel): - """Usage metadata about the response.""" + """Usage metadata about the API response. + + Parameters: + promptTokenCount: Number of tokens in the prompt. Defaults to None. + cachedContentTokenCount: Number of cached content tokens. Defaults to None. + responseTokenCount: Number of tokens in the response. Defaults to None. + toolUsePromptTokenCount: Number of tokens for tool use prompts. Defaults to None. + thoughtsTokenCount: Number of tokens for model thoughts. Defaults to None. + totalTokenCount: Total number of tokens used. Defaults to None. + promptTokensDetails: Detailed breakdown of prompt tokens by modality. Defaults to None. + cacheTokensDetails: Detailed breakdown of cache tokens by modality. Defaults to None. + responseTokensDetails: Detailed breakdown of response tokens by modality. Defaults to None. + toolUsePromptTokensDetails: Detailed breakdown of tool use tokens by modality. Defaults to None. + """ promptTokenCount: Optional[int] = None cachedContentTokenCount: Optional[int] = None @@ -270,15 +460,32 @@ class UsageMetadata(BaseModel): class ServerEvent(BaseModel): + """Server event received from the Gemini Live API. + + Parameters: + setupComplete: Setup completion notification. Defaults to None. + serverContent: Content from the server. Defaults to None. + toolCall: Tool/function call request. Defaults to None. + usageMetadata: Token usage metadata. Defaults to None. + """ + setupComplete: Optional[SetupComplete] = None serverContent: Optional[ServerContent] = None toolCall: Optional[ToolCall] = None usageMetadata: Optional[UsageMetadata] = None -def parse_server_event(message_str): - from loguru import logger # Import logger locally to avoid scoping issues + +def parse_server_event(str): + """Parse a server event from JSON string. + + Args: + str: JSON string containing the server event. + + Returns: + ServerEvent instance if parsing succeeds, None otherwise. + """ try: evt_dict = json.loads(message_str) @@ -301,7 +508,12 @@ def parse_server_event(message_str): class ContextWindowCompressionConfig(BaseModel): - """Configuration for context window compression.""" + """Configuration for context window compression. + + Parameters: + sliding_window: Whether to use sliding window compression. Defaults to True. + trigger_tokens: Token count threshold to trigger compression. Defaults to None. + """ sliding_window: Optional[bool] = Field(default=True) trigger_tokens: Optional[int] = Field(default=None) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 434008966..5303a3c6f 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google Gemini Multimodal Live API service implementation. + +This module provides real-time conversational AI capabilities using Google's +Gemini Multimodal Live API, supporting both text and audio modalities with +voice transcription, streaming responses, and tool usage. +""" + import base64 import json import time @@ -62,9 +69,10 @@ from pipecat.services.openai.llm import ( OpenAIUserContextAggregator, ) from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.string import match_endofsentence from pipecat.utils.time import time_now_iso8601 -from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt, traced_tts +from pipecat.utils.tracing.service_decorators import traced_gemini_live, traced_stt from . import events @@ -88,7 +96,11 @@ def language_to_gemini_language(language: Language) -> Optional[str]: Source: https://ai.google.dev/api/generate-content#MediaResolution - Returns None if the language is not supported by Gemini Live. + Args: + language: The language enum value to convert. + + Returns: + The Gemini language code string, or None if the language is not supported. """ language_map = { # Arabic @@ -175,8 +187,22 @@ def language_to_gemini_language(language: Language) -> Optional[str]: class GeminiMultimodalLiveContext(OpenAILLMContext): + """Extended OpenAI context for Gemini Multimodal Live API. + + Provides Gemini-specific context management including system instruction + extraction and message format conversion for the Live API. + """ + @staticmethod def upgrade(obj: OpenAILLMContext) -> "GeminiMultimodalLiveContext": + """Upgrade an OpenAI context to Gemini context. + + Args: + obj: The OpenAI context to upgrade. + + Returns: + The upgraded Gemini context instance. + """ if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GeminiMultimodalLiveContext): logger.debug(f"Upgrading to Gemini Multimodal Live Context: {obj}") obj.__class__ = GeminiMultimodalLiveContext @@ -187,6 +213,11 @@ class GeminiMultimodalLiveContext(OpenAILLMContext): pass def extract_system_instructions(self): + """Extract system instructions from context messages. + + Returns: + Combined system instruction text from all system messages. + """ system_instruction = "" for item in self.messages: if item.get("role") == "system": @@ -221,6 +252,11 @@ class GeminiMultimodalLiveContext(OpenAILLMContext): logger.info(f"Added file reference to context: {file_uri}") def get_messages_for_initializing_history(self): + """Get messages formatted for Gemini history initialization. + + Returns: + List of messages in Gemini format for conversation history. + """ messages = [] for item in self.messages: role = item.get("role") @@ -256,7 +292,19 @@ class GeminiMultimodalLiveContext(OpenAILLMContext): class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): + """User context aggregator for Gemini Multimodal Live. + + Extends OpenAI user aggregator to handle Gemini-specific message passing + while maintaining compatibility with the standard aggregation pipeline. + """ + async def process_frame(self, frame, direction): + """Process incoming frames for user context aggregation. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ await super().process_frame(frame, direction) # kind of a hack just to pass the LLMMessagesAppendFrame through, but it's fine for now if isinstance(frame, LLMMessagesAppendFrame): @@ -264,15 +312,33 @@ class GeminiMultimodalLiveUserContextAggregator(OpenAIUserContextAggregator): class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggregator): - # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, - # but the GeminiMultimodalLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We - # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames - # are process. This ensures that the context gets only one set of messages. + """Assistant context aggregator for Gemini Multimodal Live. + + Handles assistant response aggregation while filtering out LLMTextFrames + to prevent duplicate context entries, as Gemini Live pushes both + LLMTextFrames and TTSTextFrames. + """ + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames for assistant context aggregation. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the GeminiMultimodalLiveAssistantContextAggregator pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. if not isinstance(frame, LLMTextFrame): await super().process_frame(frame, direction) async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle user image frames. + + Args: + frame: The user image frame to handle. + """ # We don't want to store any images in the context. Revisit this later # when the API evolves. pass @@ -280,17 +346,41 @@ class GeminiMultimodalLiveAssistantContextAggregator(OpenAIAssistantContextAggre @dataclass class GeminiMultimodalLiveContextAggregatorPair: + """Pair of user and assistant context aggregators for Gemini Multimodal Live. + + Parameters: + _user: The user context aggregator instance. + _assistant: The assistant context aggregator instance. + """ + _user: GeminiMultimodalLiveUserContextAggregator _assistant: GeminiMultimodalLiveAssistantContextAggregator def user(self) -> GeminiMultimodalLiveUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ return self._user def assistant(self) -> GeminiMultimodalLiveAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ return self._assistant class GeminiMultimodalModalities(Enum): + """Supported modalities for Gemini Multimodal Live. + + Parameters: + TEXT: Text responses. + AUDIO: Audio responses. + """ + TEXT = "TEXT" AUDIO = "AUDIO" @@ -305,7 +395,15 @@ class GeminiMediaResolution(str, Enum): class GeminiVADParams(BaseModel): - """Voice Activity Detection parameters.""" + """Voice Activity Detection parameters for Gemini Live. + + Parameters: + disabled: Whether to disable VAD. Defaults to None. + start_sensitivity: Sensitivity for speech start detection. Defaults to None. + end_sensitivity: Sensitivity for speech end detection. Defaults to None. + prefix_padding_ms: Prefix padding in milliseconds. Defaults to None. + silence_duration_ms: Silence duration threshold in milliseconds. Defaults to None. + """ disabled: Optional[bool] = Field(default=None) start_sensitivity: Optional[events.StartSensitivity] = Field(default=None) @@ -315,7 +413,12 @@ class GeminiVADParams(BaseModel): class ContextWindowCompressionParams(BaseModel): - """Parameters for context window compression.""" + """Parameters for context window compression in Gemini Live. + + Parameters: + enabled: Whether compression is enabled. Defaults to False. + trigger_tokens: Token count to trigger compression. None uses 80% of context window. + """ enabled: bool = Field(default=False) trigger_tokens: Optional[int] = Field( @@ -324,6 +427,23 @@ class ContextWindowCompressionParams(BaseModel): class InputParams(BaseModel): + """Input parameters for Gemini Multimodal Live generation. + + Parameters: + frequency_penalty: Frequency penalty for generation (0.0-2.0). Defaults to None. + max_tokens: Maximum tokens to generate. Must be >= 1. Defaults to 4096. + presence_penalty: Presence penalty for generation (0.0-2.0). Defaults to None. + temperature: Sampling temperature (0.0-2.0). Defaults to None. + top_k: Top-k sampling parameter. Must be >= 0. Defaults to None. + top_p: Top-p sampling parameter (0.0-1.0). Defaults to None. + modalities: Response modalities. Defaults to AUDIO. + language: Language for generation. Defaults to EN_US. + media_resolution: Media resolution setting. Defaults to UNSPECIFIED. + vad: Voice activity detection parameters. Defaults to None. + context_window_compression: Context compression settings. Defaults to None. + extra: Additional parameters. Defaults to empty dict. + """ + frequency_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) max_tokens: Optional[int] = Field(default=4096, ge=1) presence_penalty: Optional[float] = Field(default=None, ge=0.0, le=2.0) @@ -348,25 +468,6 @@ class GeminiMultimodalLiveLLMService(LLMService): This service enables real-time conversations with Gemini, supporting both text and audio modalities. It handles voice transcription, streaming audio responses, and tool usage. - - Args: - api_key (str): Google AI API key - base_url (str, optional): API endpoint base URL. Defaults to - "generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent". - model (str, optional): Model identifier to use. Defaults to - "models/gemini-2.0-flash-live-001". - voice_id (str, optional): TTS voice identifier. Defaults to "Charon". - start_audio_paused (bool, optional): Whether to start with audio input paused. - Defaults to False. - start_video_paused (bool, optional): Whether to start with video input paused. - Defaults to False. - system_instruction (str, optional): System prompt for the model. Defaults to None. - tools (Union[List[dict], ToolsSchema], optional): Tools/functions available to the model. - Defaults to None. - params (InputParams, optional): Configuration parameters for the model. - Defaults to InputParams(). - inference_on_context_initialization (bool, optional): Whether to generate a response - when context is first set. Defaults to True. """ # Overriding the default adapter to use the Gemini one. @@ -388,6 +489,22 @@ class GeminiMultimodalLiveLLMService(LLMService): file_api_base_url: str = "https://generativelanguage.googleapis.com/v1beta/files", **kwargs, ): + """Initialize the Gemini Multimodal Live LLM service. + + Args: + api_key: Google AI API key for authentication. + base_url: API endpoint base URL. Defaults to the official Gemini Live endpoint. + model: Model identifier to use. Defaults to "models/gemini-2.0-flash-live-001". + voice_id: TTS voice identifier. Defaults to "Charon". + start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_video_paused: Whether to start with video input paused. Defaults to False. + system_instruction: System prompt for the model. Defaults to None. + tools: Tools/functions available to the model. Defaults to None. + params: Configuration parameters for the model. Defaults to InputParams(). + inference_on_context_initialization: Whether to generate a response when context + is first set. Defaults to True. + **kwargs: Additional arguments passed to parent LLMService. + """ super().__init__(base_url=base_url, **kwargs) params = params or InputParams() @@ -456,19 +573,43 @@ class GeminiMultimodalLiveLLMService(LLMService): self._accumulated_grounding_metadata = None def can_generate_metrics(self) -> bool: + """Check if the service can generate usage metrics. + + Returns: + True as Gemini Live supports token usage metrics. + """ return True def set_audio_input_paused(self, paused: bool): + """Set the audio input pause state. + + Args: + paused: Whether to pause audio input. + """ self._audio_input_paused = paused def set_video_input_paused(self, paused: bool): + """Set the video input pause state. + + Args: + paused: Whether to pause video input. + """ self._video_input_paused = paused def set_model_modalities(self, modalities: GeminiMultimodalModalities): + """Set the model response modalities. + + Args: + modalities: The modalities to use for responses. + """ self._settings["modalities"] = modalities def set_language(self, language: Language): - """Set the language for generation.""" + """Set the language for generation. + + Args: + language: The language to use for generation. + """ self._language = language self._language_code = language_to_gemini_language(language) or "en-US" self._settings["language"] = self._language_code @@ -481,6 +622,9 @@ class GeminiMultimodalLiveLLMService(LLMService): way to trigger the pipeline. This sends the history to the server. The `inference_on_context_initialization` flag controls whether to set the turnComplete flag when we do this. Without that flag, the model will not respond. This is often what we want when setting the context at the beginning of a conversation. + + Args: + context: The OpenAI LLM context to set. """ if self._context: logger.error( @@ -495,14 +639,29 @@ class GeminiMultimodalLiveLLMService(LLMService): # async def start(self, frame: StartFrame): + """Start the service and establish websocket connection. + + Args: + frame: The start frame. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the service and close connections. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the service and close connections. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() @@ -537,6 +696,12 @@ class GeminiMultimodalLiveLLMService(LLMService): # async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames for the Gemini Live service. + + Args: + frame: The frame to process. + direction: The frame processing direction. + """ await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -592,6 +757,11 @@ class GeminiMultimodalLiveLLMService(LLMService): # async def send_client_event(self, event): + """Send a client event to the Gemini Live API. + + Args: + event: The event to send. + """ await self._ws_send(event.model_dump(exclude_none=True)) async def _connect(self): @@ -735,9 +905,7 @@ class GeminiMultimodalLiveLLMService(LLMService): # async def _receive_task_handler(self): - async for message in self._websocket: - self.start_watchdog() - + async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager): evt = events.parse_server_event(message) # logger.debug(f"Received event: {message[:500]}") # logger.debug(f"Received event: {evt}") @@ -767,8 +935,6 @@ class GeminiMultimodalLiveLLMService(LLMService): pass - self.reset_watchdog() - # # # @@ -1186,22 +1352,19 @@ class GeminiMultimodalLiveLLMService(LLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> GeminiMultimodalLiveContextAggregatorPair: - """Create an instance of GeminiMultimodalLiveContextAggregatorPair from - an OpenAILLMContext. Constructor keyword arguments for both the user and - assistant aggregators can be provided. + """Create an instance of GeminiMultimodalLiveContextAggregatorPair from an OpenAILLMContext. + + Constructor keyword arguments for both the user and assistant aggregators can be provided. Args: - context (OpenAILLMContext): The LLM context. - user_params (LLMUserAggregatorParams, optional): User aggregator - parameters. - assistant_params (LLMAssistantAggregatorParams, optional): User - aggregator parameters. + context: The LLM context to use. + user_params: User aggregator parameters. Defaults to LLMUserAggregatorParams(). + assistant_params: Assistant aggregator parameters. Defaults to LLMAssistantAggregatorParams(). Returns: GeminiMultimodalLiveContextAggregatorPair: A pair of context aggregators, one for the user and one for the assistant, encapsulated in an GeminiMultimodalLiveContextAggregatorPair. - """ context.set_llm_adapter(self.get_llm_adapter()) diff --git a/src/pipecat/services/gladia/config.py b/src/pipecat/services/gladia/config.py index 662996f1d..0af008773 100644 --- a/src/pipecat/services/gladia/config.py +++ b/src/pipecat/services/gladia/config.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Configuration for the Gladia STT service.""" + from typing import Any, Dict, List, Optional, Union from pydantic import BaseModel @@ -14,7 +16,7 @@ from pipecat.transcriptions.language import Language class LanguageConfig(BaseModel): """Configuration for language detection and handling. - Attributes: + Parameters: languages: List of language codes to use for transcription code_switching: Whether to auto-detect language changes during transcription """ @@ -26,7 +28,7 @@ class LanguageConfig(BaseModel): class PreProcessingConfig(BaseModel): """Configuration for audio pre-processing options. - Attributes: + Parameters: speech_threshold: Sensitivity for speech detection (0-1) """ @@ -36,7 +38,7 @@ class PreProcessingConfig(BaseModel): class CustomVocabularyItem(BaseModel): """Represents a custom vocabulary item with an intensity value. - Attributes: + Parameters: value: The vocabulary word or phrase intensity: The bias intensity for this vocabulary item (0-1) """ @@ -48,7 +50,7 @@ class CustomVocabularyItem(BaseModel): class CustomVocabularyConfig(BaseModel): """Configuration for custom vocabulary. - Attributes: + Parameters: vocabulary: List of words/phrases or CustomVocabularyItem objects default_intensity: Default intensity for simple string vocabulary items """ @@ -60,7 +62,7 @@ class CustomVocabularyConfig(BaseModel): class CustomSpellingConfig(BaseModel): """Configuration for custom spelling rules. - Attributes: + Parameters: spelling_dictionary: Mapping of correct spellings to phonetic variations """ @@ -70,7 +72,7 @@ class CustomSpellingConfig(BaseModel): class TranslationConfig(BaseModel): """Configuration for real-time translation. - Attributes: + Parameters: target_languages: List of target language codes for translation model: Translation model to use ("base" or "enhanced") match_original_utterances: Whether to align translations with original utterances @@ -92,7 +94,7 @@ class TranslationConfig(BaseModel): class RealtimeProcessingConfig(BaseModel): """Configuration for real-time processing features. - Attributes: + Parameters: words_accurate_timestamps: Whether to provide per-word timestamps custom_vocabulary: Whether to enable custom vocabulary custom_vocabulary_config: Custom vocabulary configuration @@ -118,7 +120,7 @@ class RealtimeProcessingConfig(BaseModel): class MessagesConfig(BaseModel): """Configuration for controlling which message types are sent via WebSocket. - Attributes: + Parameters: receive_partial_transcripts: Whether to receive intermediate transcription results receive_final_transcripts: Whether to receive final transcription results receive_speech_events: Whether to receive speech begin/end events @@ -144,7 +146,7 @@ class MessagesConfig(BaseModel): class GladiaInputParams(BaseModel): """Configuration parameters for the Gladia STT service. - Attributes: + Parameters: encoding: Audio encoding format bit_depth: Audio bit depth channels: Number of audio channels diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index 885fe8dc2..c436a7ea9 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Gladia Speech-to-Text (STT) service implementation. + +This module provides a Speech-to-Text service using Gladia's real-time WebSocket API, +supporting multiple languages, custom vocabulary, and various audio processing options. +""" + import asyncio import base64 import json @@ -25,6 +31,7 @@ from pipecat.frames.frames import ( from pipecat.services.gladia.config import GladiaInputParams from pipecat.services.stt_service import STTService from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt @@ -40,10 +47,10 @@ def language_to_gladia_language(language: Language) -> Optional[str]: """Convert a Language enum to Gladia's language code format. Args: - language: The Language enum value to convert + language: The Language enum value to convert. Returns: - The Gladia language code string or None if not supported + The Gladia language code string or None if not supported. """ BASE_LANGUAGES = { Language.AF: "af", @@ -179,6 +186,7 @@ class GladiaSTTService(STTService): This service connects to Gladia's WebSocket API for real-time transcription with support for multiple languages, custom vocabulary, and various processing options. + Provides automatic reconnection, audio buffering, and comprehensive error handling. For complete API documentation, see: https://docs.gladia.io/api-reference/v2/live/init """ @@ -203,16 +211,16 @@ class GladiaSTTService(STTService): """Initialize the Gladia STT service. Args: - api_key: Gladia API key - url: Gladia API URL - confidence: Minimum confidence threshold for transcriptions - sample_rate: Audio sample rate in Hz - model: Model to use ("solaria-1") - params: Additional configuration parameters - max_reconnection_attempts: Maximum number of reconnection attempts - reconnection_delay: Initial delay between reconnection attempts (exponential backoff) - max_buffer_size: Maximum size of audio buffer in bytes - **kwargs: Additional arguments passed to the STTService + api_key: Gladia API key for authentication. + url: Gladia API URL. Defaults to "https://api.gladia.io/v2/live". + confidence: Minimum confidence threshold for transcriptions (0.0-1.0). + sample_rate: Audio sample rate in Hz. If None, uses service default. + model: Model to use for transcription. Defaults to "solaria-1". + params: Additional configuration parameters for Gladia service. + max_reconnection_attempts: Maximum number of reconnection attempts. Defaults to 5. + reconnection_delay: Initial delay between reconnection attempts in seconds. + max_buffer_size: Maximum size of audio buffer in bytes. Defaults to 20MB. + **kwargs: Additional arguments passed to the STTService parent class. """ super().__init__(sample_rate=sample_rate, **kwargs) @@ -255,10 +263,22 @@ class GladiaSTTService(STTService): self._should_reconnect = True def can_generate_metrics(self) -> bool: + """Check if the service can generate performance metrics. + + Returns: + True, indicating this service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert pipecat Language enum to Gladia's language code.""" + """Convert pipecat Language enum to Gladia's language code. + + Args: + language: The Language enum value to convert. + + Returns: + The Gladia language code string or None if not supported. + """ return language_to_gladia_language(language) def _prepare_settings(self) -> Dict[str, Any]: @@ -313,7 +333,11 @@ class GladiaSTTService(STTService): return settings async def start(self, frame: StartFrame): - """Start the Gladia STT websocket connection.""" + """Start the Gladia STT websocket connection. + + Args: + frame: The start frame triggering service startup. + """ await super().start(frame) if self._connection_task: return @@ -322,7 +346,11 @@ class GladiaSTTService(STTService): self._connection_task = self.create_task(self._connection_handler()) async def stop(self, frame: EndFrame): - """Stop the Gladia STT websocket connection.""" + """Stop the Gladia STT websocket connection. + + Args: + frame: The end frame triggering service shutdown. + """ await super().stop(frame) self._should_reconnect = False await self._send_stop_recording() @@ -334,7 +362,11 @@ class GladiaSTTService(STTService): await self._cleanup_connection() async def cancel(self, frame: CancelFrame): - """Cancel the Gladia STT websocket connection.""" + """Cancel the Gladia STT websocket connection. + + Args: + frame: The cancel frame triggering service cancellation. + """ await super().cancel(frame) self._should_reconnect = False @@ -345,7 +377,14 @@ class GladiaSTTService(STTService): await self._cleanup_connection() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Run speech-to-text on audio data.""" + """Run speech-to-text on audio data. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None (processing is handled asynchronously via WebSocket). + """ await self.start_ttfb_metrics() await self.start_processing_metrics() @@ -391,8 +430,8 @@ class GladiaSTTService(STTService): await self._send_buffered_audio() # Start tasks - self._receive_task = asyncio.create_task(self._receive_task_handler()) - self._keepalive_task = asyncio.create_task(self._keepalive_task_handler()) + self._receive_task = self.create_task(self._receive_task_handler()) + self._keepalive_task = self.create_task(self._keepalive_task_handler()) # Wait for tasks to complete await asyncio.gather(self._receive_task, self._keepalive_task) @@ -403,9 +442,9 @@ class GladiaSTTService(STTService): # Clean up tasks if self._receive_task: - self._receive_task.cancel() + await self.cancel_task(self._receive_task) if self._keepalive_task: - self._keepalive_task.cancel() + await self.cancel_task(self._keepalive_task) # Attempt reconnect using helper if not await self._maybe_reconnect(): @@ -484,9 +523,11 @@ class GladiaSTTService(STTService): async def _keepalive_task_handler(self): """Send periodic empty audio chunks to keep the connection alive.""" try: + KEEPALIVE_SLEEP = 20 if self.task_manager.task_watchdog_enabled else 3 while self._connection_active: - # Send keepalive every 20 seconds (Gladia times out after 30 seconds) - await asyncio.sleep(20) + self.reset_watchdog() + # Send keepalive (Gladia times out after 30 seconds) + await asyncio.sleep(KEEPALIVE_SLEEP) if self._websocket and not self._websocket.closed: # Send an empty audio chunk as keepalive empty_audio = b"" @@ -501,9 +542,7 @@ class GladiaSTTService(STTService): async def _receive_task_handler(self): try: - async for message in self._websocket: - self.start_watchdog() - + async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager): content = json.loads(message) # Handle audio chunk acknowledgments @@ -568,8 +607,6 @@ class GladiaSTTService(STTService): pass except Exception as e: logger.error(f"Error in Gladia WebSocket handler: {e}") - finally: - self.reset_watchdog() async def _maybe_reconnect(self) -> bool: """Handle exponential backoff reconnection logic.""" diff --git a/src/pipecat/services/google/frames.py b/src/pipecat/services/google/frames.py index 700a39a7e..bc174937a 100644 --- a/src/pipecat/services/google/frames.py +++ b/src/pipecat/services/google/frames.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google AI service frames for search and grounding functionality. + +This module defines specialized frame types for handling search results +and grounding metadata from Google AI models, particularly for Gemini +models that support web search and fact grounding capabilities. +""" + from dataclasses import dataclass, field from typing import List, Optional @@ -12,12 +19,27 @@ from pipecat.frames.frames import DataFrame @dataclass class LLMSearchResult: + """Represents a single search result with confidence scores. + + Parameters: + text: The search result text content. + confidence: List of confidence scores associated with the result. + """ + text: str confidence: List[float] = field(default_factory=list) @dataclass class LLMSearchOrigin: + """Represents the origin source of search results. + + Parameters: + site_uri: URI of the source website. + site_title: Title of the source website. + results: List of search results from this origin. + """ + site_uri: Optional[str] = None site_title: Optional[str] = None results: List[LLMSearchResult] = field(default_factory=list) @@ -25,9 +47,27 @@ class LLMSearchOrigin: @dataclass class LLMSearchResponseFrame(DataFrame): + """Frame containing search results and grounding information from Google AI models. + + This frame is used to convey search results and grounding metadata + from Google AI models that support web search capabilities. It includes + the search result text, rendered content, and detailed origin information + with confidence scores. + + Parameters: + search_result: The main search result text. + rendered_content: Rendered content from the search entry point. + origins: List of search result origins with detailed information. + """ + search_result: Optional[str] = None rendered_content: Optional[str] = None origins: List[LLMSearchOrigin] = field(default_factory=list) def __str__(self): + """Return string representation of the search response frame. + + Returns: + String representation showing search result and origins. + """ return f"LLMSearchResponseFrame(search_result={self.search_result}, origins={self.origins})" diff --git a/src/pipecat/services/google/image.py b/src/pipecat/services/google/image.py index dc0218b8c..5d7a461b3 100644 --- a/src/pipecat/services/google/image.py +++ b/src/pipecat/services/google/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google AI image generation service implementation. + +This module provides integration with Google's Imagen model for generating +images from text prompts using the Google AI API. +""" + import io import os @@ -29,7 +35,22 @@ except ModuleNotFoundError as e: class GoogleImageGenService(ImageGenService): + """Google AI image generation service using Imagen models. + + Provides text-to-image generation capabilities using Google's Imagen models + through the Google AI API. Supports multiple image generation and negative + prompting for enhanced control over generated content. + """ + class InputParams(BaseModel): + """Configuration parameters for Google image generation. + + Parameters: + number_of_images: Number of images to generate (1-8). Defaults to 1. + model: Google Imagen model to use. Defaults to "imagen-3.0-generate-002". + negative_prompt: Optional negative prompt to guide what not to include. + """ + number_of_images: int = Field(default=1, ge=1, le=8) model: str = Field(default="imagen-3.0-generate-002") negative_prompt: Optional[str] = Field(default=None) @@ -41,22 +62,38 @@ class GoogleImageGenService(ImageGenService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the GoogleImageGenService with API key and parameters. + + Args: + api_key: Google AI API key for authentication. + params: Configuration parameters for image generation. Defaults to InputParams(). + **kwargs: Additional arguments passed to the parent ImageGenService. + """ super().__init__(**kwargs) self._params = params or GoogleImageGenService.InputParams() self._client = genai.Client(api_key=api_key) self.set_model_name(self._params.model) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Google image generation service supports metrics. + """ return True async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: """Generate images from a text prompt using Google's Imagen model. Args: - prompt (str): The text description to generate images from. + prompt: The text description to generate images from. Yields: - Frame: Generated image frames or error frames. + Frame: Generated URLImageRawFrame objects containing the generated + images, or ErrorFrame objects if generation fails. + + Raises: + Exception: If there are issues with the Google AI API or image processing. """ logger.debug(f"Generating image from prompt: {prompt}") await self.start_ttfb_metrics() diff --git a/src/pipecat/services/google/llm.py b/src/pipecat/services/google/llm.py index f983b7342..86ed4dd88 100644 --- a/src/pipecat/services/google/llm.py +++ b/src/pipecat/services/google/llm.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google Gemini integration for Pipecat. + +This module provides Google Gemini integration for the Pipecat framework, +including LLM services, context management, and message aggregation. +""" + import base64 import io import json @@ -47,6 +53,7 @@ from pipecat.services.openai.llm import ( OpenAIAssistantContextAggregator, OpenAIUserContextAggregator, ) +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.tracing.service_decorators import traced_llm # Suppress gRPC fork warnings @@ -70,7 +77,14 @@ except ModuleNotFoundError as e: class GoogleUserContextAggregator(OpenAIUserContextAggregator): + """Google-specific user context aggregator. + + Extends OpenAI user context aggregator to handle Google AI's specific + Content and Part message format for user messages. + """ + async def push_aggregation(self): + """Push aggregated user text as a Google Content message.""" if len(self._aggregation) > 0: self._context.add_message(Content(role="user", parts=[Part(text=self._aggregation)])) @@ -87,10 +101,26 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator): class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Google-specific assistant context aggregator. + + Extends OpenAI assistant context aggregator to handle Google AI's specific + Content and Part message format for assistant responses and function calls. + """ + async def handle_aggregation(self, aggregation: str): + """Handle aggregated assistant text response. + + Args: + aggregation: The aggregated text response from the assistant. + """ self._context.add_message(Content(role="model", parts=[Part(text=aggregation)])) async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + """Handle function call in progress frame. + + Args: + frame: Frame containing function call details. + """ self._context.add_message( Content( role="model", @@ -119,6 +149,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call result frame. + + Args: + frame: Frame containing function call result. + """ if frame.result: await self._update_function_call_result( frame.function_name, frame.tool_call_id, frame.result @@ -129,6 +164,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + """Handle function call cancellation frame. + + Args: + frame: Frame containing function call cancellation details. + """ await self._update_function_call_result( frame.function_name, frame.tool_call_id, "CANCELLED" ) @@ -143,6 +183,11 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): part.function_response.response = {"value": json.dumps(result)} async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle user image frame. + + Args: + frame: Frame containing user image data and request context. + """ await self._update_function_call_result( frame.request.function_name, frame.request.tool_call_id, "COMPLETED" ) @@ -156,28 +201,66 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): @dataclass class GoogleContextAggregatorPair: + """Pair of Google context aggregators for user and assistant messages. + + Parameters: + _user: User context aggregator for handling user messages. + _assistant: Assistant context aggregator for handling assistant responses. + """ + _user: GoogleUserContextAggregator _assistant: GoogleAssistantContextAggregator def user(self) -> GoogleUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ return self._user def assistant(self) -> GoogleAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ return self._assistant class GoogleLLMContext(OpenAILLMContext): + """Google AI LLM context that extends OpenAI context for Google-specific formatting. + + This class handles conversion between OpenAI-style messages and Google AI's + Content/Part format, including system messages, function calls, and media. + """ + def __init__( self, messages: Optional[List[dict]] = None, tools: Optional[List[dict]] = None, tool_choice: Optional[dict] = None, ): + """Initialize GoogleLLMContext. + + Args: + messages: Initial messages in OpenAI format. + tools: Available tools/functions for the model. + tool_choice: Tool choice configuration. + """ super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) self.system_message = None @staticmethod def upgrade_to_google(obj: OpenAILLMContext) -> "GoogleLLMContext": + """Upgrade an OpenAI context to a Google context. + + Args: + obj: OpenAI LLM context to upgrade. + + Returns: + GoogleLLMContext instance with converted messages. + """ if isinstance(obj, OpenAILLMContext) and not isinstance(obj, GoogleLLMContext): logger.debug(f"Upgrading to Google: {obj}") obj.__class__ = GoogleLLMContext @@ -185,10 +268,20 @@ class GoogleLLMContext(OpenAILLMContext): return obj def set_messages(self, messages: List): + """Set messages and restructure them for Google format. + + Args: + messages: List of messages to set. + """ self._messages[:] = messages self._restructure_from_openai_messages() def add_messages(self, messages: List): + """Add messages to the context, converting to Google format as needed. + + Args: + messages: List of messages to add (can be mixed formats). + """ # Convert each message individually converted_messages = [] for msg in messages: @@ -205,6 +298,11 @@ class GoogleLLMContext(OpenAILLMContext): self._messages.extend(converted_messages) def get_messages_for_logging(self): + """Get messages formatted for logging with sensitive data redacted. + + Returns: + List of message dictionaries with inline data redacted. + """ msgs = [] for message in self.messages: obj = message.to_json_dict() @@ -221,6 +319,14 @@ class GoogleLLMContext(OpenAILLMContext): def add_image_frame_message( self, *, format: str, size: tuple[int, int], image: bytes, text: str = None ): + """Add an image message to the context. + + Args: + format: Image format (e.g., 'RGB', 'RGBA'). + size: Image dimensions as (width, height). + image: Raw image bytes. + text: Optional text to accompany the image. + """ buffer = io.BytesIO() Image.frombytes(format, size, image).save(buffer, format="JPEG") @@ -234,6 +340,12 @@ class GoogleLLMContext(OpenAILLMContext): def add_audio_frames_message( self, *, audio_frames: list[AudioRawFrame], text: str = "Audio follows" ): + """Add audio frames as a message to the context. + + Args: + audio_frames: List of audio frames to add. + text: Text description of the audio content. + """ if not audio_frames: return @@ -447,17 +559,28 @@ class GoogleLLMContext(OpenAILLMContext): class GoogleLLMService(LLMService): - """This class implements inference with Google's AI models. + """Google AI (Gemini) LLM service implementation. - This service translates internally from OpenAILLMContext to the messages format - expected by the Google AI model. We are using the OpenAILLMContext as a lingua - franca for all LLM services, so that it is easy to switch between different LLMs. + This class implements inference with Google's AI models, translating internally + from OpenAILLMContext to the messages format expected by the Google AI model. + We use OpenAILLMContext as a lingua franca for all LLM services to enable + easy switching between different LLMs. """ # Overriding the default adapter to use the Gemini one. adapter_class = GeminiLLMAdapter class InputParams(BaseModel): + """Input parameters for Google AI models. + + Parameters: + max_tokens: Maximum number of tokens to generate. + temperature: Sampling temperature between 0.0 and 2.0. + top_k: Top-k sampling parameter. + top_p: Top-p sampling parameter between 0.0 and 1.0. + extra: Additional parameters as a dictionary. + """ + max_tokens: Optional[int] = Field(default=4096, ge=1) temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0) top_k: Optional[int] = Field(default=None, ge=0) @@ -475,6 +598,17 @@ class GoogleLLMService(LLMService): tool_config: Optional[Dict[str, Any]] = None, **kwargs, ): + """Initialize the Google LLM service. + + Args: + api_key: Google AI API key for authentication. + model: Model name to use. Defaults to "gemini-2.0-flash". + params: Input parameters for the model. + system_instruction: System instruction/prompt for the model. + tools: List of available tools/functions. + tool_config: Configuration for tool usage. + **kwargs: Additional arguments passed to parent class. + """ super().__init__(**kwargs) params = params or GoogleLLMService.InputParams() @@ -494,11 +628,30 @@ class GoogleLLMService(LLMService): self._tool_config = tool_config def can_generate_metrics(self) -> bool: + """Check if the service can generate usage metrics. + + Returns: + True, as Google AI provides token usage metrics. + """ return True def _create_client(self, api_key: str): self._client = genai.Client(api_key=api_key) + def _maybe_unset_thinking_budget(self, generation_params: Dict[str, Any]): + try: + # There's no way to introspect on model capabilities, so + # to check for models that we know default to thinkin on + # and can be configured to turn it off. + if not self._model_name.startswith("gemini-2.5-flash"): + return + # If thinking_config is already set, don't override it. + if "thinking_config" in generation_params: + return + generation_params.setdefault("thinking_config", {})["thinking_budget"] = 0 + except Exception as e: + logger.exception(f"Failed to unset thinking budget: {e}") + @traced_llm async def _process_context(self, context: OpenAILLMContext): await self.push_frame(LLMFullResponseStartFrame()) @@ -506,6 +659,8 @@ class GoogleLLMService(LLMService): prompt_tokens = 0 completion_tokens = 0 total_tokens = 0 + cache_read_input_tokens = 0 + reasoning_tokens = 0 grounding_metadata = None search_result = "" @@ -545,6 +700,12 @@ class GoogleLLMService(LLMService): if v is not None } + if self._settings["extra"]: + generation_params.update(self._settings["extra"]) + + # possibly modify generation_params (in place) to set thinking to off by default + self._maybe_unset_thinking_budget(generation_params) + generation_config = ( GenerateContentConfig(**generation_params) if generation_params else None ) @@ -557,13 +718,15 @@ class GoogleLLMService(LLMService): ) function_calls = [] - async for chunk in response: + async for chunk in WatchdogAsyncIterator(response, manager=self.task_manager): # Stop TTFB metrics after the first chunk await self.stop_ttfb_metrics() if chunk.usage_metadata: prompt_tokens += chunk.usage_metadata.prompt_token_count or 0 completion_tokens += chunk.usage_metadata.candidates_token_count or 0 total_tokens += chunk.usage_metadata.total_token_count or 0 + cache_read_input_tokens += chunk.usage_metadata.cached_content_token_count or 0 + reasoning_tokens += chunk.usage_metadata.thoughts_token_count or 0 if not chunk.candidates: continue @@ -645,11 +808,19 @@ class GoogleLLMService(LLMService): prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, + cache_read_input_tokens=cache_read_input_tokens, + reasoning_tokens=reasoning_tokens, ) ) await self.push_frame(LLMFullResponseEndFrame()) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle different frame types. + + Args: + frame: The frame to process. + direction: Direction of frame processing. + """ await super().process_frame(frame, direction) context = None @@ -678,16 +849,15 @@ class GoogleLLMService(LLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> GoogleContextAggregatorPair: - """Create an instance of GoogleContextAggregatorPair from an - OpenAILLMContext. Constructor keyword arguments for both the user and - assistant aggregators can be provided. + """Create Google-specific context aggregators. + + Creates a pair of context aggregators optimized for Google's message format, + including support for function calls, tool usage, and image handling. Args: - context (OpenAILLMContext): The LLM context. - user_params (LLMUserAggregatorParams, optional): User aggregator - parameters. - assistant_params (LLMAssistantAggregatorParams, optional): User - aggregator parameters. + context: The LLM context to create aggregators for. + user_params: Parameters for user message aggregation. + assistant_params: Parameters for assistant message aggregation. Returns: GoogleContextAggregatorPair: A pair of context aggregators, one for diff --git a/src/pipecat/services/google/llm_openai.py b/src/pipecat/services/google/llm_openai.py index a497cb229..02c50f885 100644 --- a/src/pipecat/services/google/llm_openai.py +++ b/src/pipecat/services/google/llm_openai.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google LLM service using OpenAI-compatible API format. + +This module provides integration with Google's AI LLM models using the OpenAI +API format through Google's Gemini API OpenAI compatibility layer. +""" + import json import os @@ -11,6 +17,7 @@ from openai import AsyncStream from openai.types.chat import ChatCompletionChunk from pipecat.services.llm_service import FunctionCallFromLLM +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator # Suppress gRPC fork warnings os.environ["GRPC_ENABLE_FORK_SUPPORT"] = "false" @@ -24,8 +31,17 @@ from pipecat.services.openai.llm import OpenAILLMService class GoogleLLMOpenAIBetaService(OpenAILLMService): - """This class implements inference with Google's AI LLM models using the OpenAI format. - Ref - https://ai.google.dev/gemini-api/docs/openai + """Google LLM service using OpenAI-compatible API format. + + This service provides access to Google's AI LLM models (like Gemini) through + the OpenAI API format. It handles streaming responses, function calls, and + tool usage while maintaining compatibility with OpenAI's interface. + + Note: This service includes a workaround for a Google API bug where function + call indices may be incorrectly set to None, resulting in empty function names. + + Reference: + https://ai.google.dev/gemini-api/docs/openai """ def __init__( @@ -36,6 +52,14 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): model: str = "gemini-2.0-flash", **kwargs, ): + """Initialize the Google LLM service. + + Args: + api_key: Google API key for authentication. + base_url: Base URL for Google's OpenAI-compatible API. + model: Google model name to use (e.g., "gemini-2.0-flash"). + **kwargs: Additional arguments passed to the parent OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) async def _process_context(self, context: OpenAILLMContext): @@ -53,7 +77,7 @@ class GoogleLLMOpenAIBetaService(OpenAILLMService): context ) - async for chunk in chunk_stream: + async for chunk in WatchdogAsyncIterator(chunk_stream, manager=self.task_manager): if chunk.usage: tokens = LLMTokenUsage( prompt_tokens=chunk.usage.prompt_tokens, diff --git a/src/pipecat/services/google/llm_vertex.py b/src/pipecat/services/google/llm_vertex.py index 9b0ed6501..22b6258a5 100644 --- a/src/pipecat/services/google/llm_vertex.py +++ b/src/pipecat/services/google/llm_vertex.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google Vertex AI LLM service implementation. + +This module provides integration with Google's AI models via Vertex AI while +maintaining OpenAI API compatibility through Google's OpenAI-compatible endpoint. +""" + import json import os @@ -31,16 +37,24 @@ except ModuleNotFoundError as e: class GoogleVertexLLMService(OpenAILLMService): - """Implements inference with Google's AI models via Vertex AI while - maintaining OpenAI API compatibility. + """Google Vertex AI LLM service with OpenAI API compatibility. + + Provides access to Google's AI models via Vertex AI while maintaining + OpenAI API compatibility. Handles authentication using Google service + account credentials and constructs appropriate endpoint URLs for + different GCP regions and projects. Reference: - https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library - + https://cloud.google.com/vertex-ai/generative-ai/docs/multimodal/call-vertex-using-openai-library """ class InputParams(OpenAILLMService.InputParams): - """Input parameters specific to Vertex AI.""" + """Input parameters specific to Vertex AI. + + Parameters: + location: GCP region for Vertex AI endpoint (e.g., "us-east4"). + project_id: Google Cloud project ID. + """ # https://cloud.google.com/vertex-ai/generative-ai/docs/learn/locations location: str = "us-east4" @@ -58,11 +72,11 @@ class GoogleVertexLLMService(OpenAILLMService): """Initializes the VertexLLMService. Args: - credentials (Optional[str]): JSON string of service account credentials. - credentials_path (Optional[str]): Path to the service account JSON file. - model (str): Model identifier. Defaults to "google/gemini-2.0-flash-001". - params (InputParams): Vertex AI input parameters. - **kwargs: Additional arguments for OpenAILLMService. + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. + model: Model identifier (e.g., "google/gemini-2.0-flash-001"). + params: Vertex AI input parameters including location and project. + **kwargs: Additional arguments passed to OpenAILLMService. """ params = params or OpenAILLMService.InputParams() base_url = self._get_base_url(params) @@ -74,7 +88,7 @@ class GoogleVertexLLMService(OpenAILLMService): @staticmethod def _get_base_url(params: InputParams) -> str: - """Constructs the base URL for Vertex AI API.""" + """Construct the base URL for Vertex AI API.""" return ( f"https://{params.location}-aiplatform.googleapis.com/v1/" f"projects/{params.project_id}/locations/{params.location}/endpoints/openapi" @@ -82,14 +96,22 @@ class GoogleVertexLLMService(OpenAILLMService): @staticmethod def _get_api_token(credentials: Optional[str], credentials_path: Optional[str]) -> str: - """Retrieves an authentication token using Google service account credentials. + """Retrieve an authentication token using Google service account credentials. + + Supports multiple authentication methods: + 1. Direct JSON credentials string + 2. Path to service account JSON file + 3. Default application credentials (ADC) Args: - credentials (Optional[str]): JSON string of service account credentials. - credentials_path (Optional[str]): Path to the service account JSON file. + credentials: JSON string of service account credentials. + credentials_path: Path to the service account JSON file. Returns: - str: OAuth token for API authentication. + OAuth token for API authentication. + + Raises: + ValueError: If no valid credentials are provided or found. """ creds: Optional[service_account.Credentials] = None diff --git a/src/pipecat/services/google/rtvi.py b/src/pipecat/services/google/rtvi.py index cd60f6f1f..3031ad532 100644 --- a/src/pipecat/services/google/rtvi.py +++ b/src/pipecat/services/google/rtvi.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google RTVI integration models and observer implementation. + +This module provides integration with Google's services through the RTVI framework, +including models for search responses and an observer for handling Google-specific +frame types. +""" + from typing import List, Literal, Optional from pydantic import BaseModel @@ -16,22 +23,56 @@ from pipecat.services.google.frames import LLMSearchOrigin, LLMSearchResponseFra class RTVISearchResponseMessageData(BaseModel): + """Data payload for search response messages in RTVI protocol. + + Parameters: + search_result: The search result text, if available. + rendered_content: The rendered content from the search, if available. + origins: List of search result origins with metadata. + """ + search_result: Optional[str] rendered_content: Optional[str] origins: List[LLMSearchOrigin] class RTVIBotLLMSearchResponseMessage(BaseModel): + """RTVI message for bot LLM search responses. + + Parameters: + label: Always "rtvi-ai" for RTVI protocol messages. + type: Always "bot-llm-search-response" for this message type. + data: The search response data payload. + """ + label: Literal["rtvi-ai"] = "rtvi-ai" type: Literal["bot-llm-search-response"] = "bot-llm-search-response" data: RTVISearchResponseMessageData class GoogleRTVIObserver(RTVIObserver): + """RTVI observer for Google service integration. + + Extends the base RTVIObserver to handle Google-specific frame types, + particularly LLM search response frames from Google services. + """ + def __init__(self, rtvi: RTVIProcessor): + """Initialize the Google RTVI observer. + + Args: + rtvi: The RTVI processor to send messages through. + """ super().__init__(rtvi) async def on_push_frame(self, data: FramePushed): + """Process frames being pushed through the pipeline. + + Handles Google-specific frames in addition to the base RTVI frame types. + + Args: + data: Frame push event data containing frame and metadata. + """ await super().on_push_frame(data) frame = data.frame diff --git a/src/pipecat/services/google/stt.py b/src/pipecat/services/google/stt.py index 26186e6bf..e94fbbb12 100644 --- a/src/pipecat/services/google/stt.py +++ b/src/pipecat/services/google/stt.py @@ -4,11 +4,19 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Google Cloud Speech-to-Text V2 service implementation for Pipecat. + +This module provides a Google Cloud Speech-to-Text V2 service with streaming +support, enabling real-time speech recognition with features like automatic +punctuation, voice activity detection, and multi-language support. +""" + import asyncio import json import os import time +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.tracing.service_decorators import traced_stt # Suppress gRPC fork warnings @@ -352,9 +360,15 @@ class GoogleSTTService(STTService): Provides real-time speech recognition using Google Cloud's Speech-to-Text V2 API with streaming support. Handles audio transcription and optional voice activity detection. + Implements automatic stream reconnection to handle Google's 4-minute streaming limit. Attributes: InputParams: Configuration parameters for the STT service. + STREAMING_LIMIT: Google Cloud's streaming limit in milliseconds (4 minutes). + + Raises: + ValueError: If neither credentials nor credentials_path is provided. + ValueError: If project ID is not found in credentials. """ # Google Cloud's STT service has a connection time limit of 5 minutes per stream. @@ -366,7 +380,7 @@ class GoogleSTTService(STTService): class InputParams(BaseModel): """Configuration parameters for Google Speech-to-Text. - Attributes: + Parameters: languages: Single language or list of recognition languages. First language is primary. model: Speech recognition model to use. use_separate_recognition_per_channel: Process each audio channel separately. @@ -395,13 +409,25 @@ class GoogleSTTService(STTService): @field_validator("languages", mode="before") @classmethod def validate_languages(cls, v) -> List[Language]: + """Ensure languages is always a list. + + Args: + v: Single Language enum or list of Language enums. + + Returns: + List[Language]: List of configured languages. + """ if isinstance(v, Language): return [v] return v @property def language_list(self) -> List[Language]: - """Get languages as a guaranteed list.""" + """Get languages as a guaranteed list. + + Returns: + List[Language]: List of configured languages. + """ assert isinstance(self.languages, list) return self.languages @@ -424,10 +450,6 @@ class GoogleSTTService(STTService): sample_rate: Audio sample rate in Hertz. params: Configuration parameters for the service. **kwargs: Additional arguments passed to STTService. - - Raises: - ValueError: If neither credentials nor credentials_path is provided. - ValueError: If project ID is not found in credentials. """ super().__init__(sample_rate=sample_rate, **kwargs) @@ -436,7 +458,6 @@ class GoogleSTTService(STTService): self._location = location self._stream = None self._config = None - self._request_queue = asyncio.Queue() self._streaming_task = None # Used for keep-alive logic @@ -501,6 +522,11 @@ class GoogleSTTService(STTService): } def can_generate_metrics(self) -> bool: + """Check if the service can generate metrics. + + Returns: + bool: True, as this service supports metrics generation. + """ return True def language_to_service_language(self, language: Language | List[Language]) -> str | List[str]: @@ -548,7 +574,11 @@ class GoogleSTTService(STTService): await self._reconnect_if_needed() async def set_model(self, model: str): - """Update the service's recognition model.""" + """Update the service's recognition model. + + Args: + model: The new recognition model to use. + """ logger.debug(f"Switching STT model to: {model}") await super().set_model(model) self._settings["model"] = model @@ -556,14 +586,29 @@ class GoogleSTTService(STTService): await self._reconnect_if_needed() async def start(self, frame: StartFrame): + """Start the STT service and establish connection. + + Args: + frame: The start frame triggering the service start. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the STT service and clean up resources. + + Args: + frame: The end frame triggering the service stop. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the STT service and clean up resources. + + Args: + frame: The cancel frame triggering the service cancellation. + """ await super().cancel(frame) await self._disconnect() @@ -585,7 +630,7 @@ class GoogleSTTService(STTService): """Update service options dynamically. Args: - languages: New list of recongition languages. + languages: New list of recognition languages. model: New recognition model. enable_automatic_punctuation: Enable/disable automatic punctuation. enable_spoken_punctuation: Enable/disable spoken punctuation. @@ -683,23 +728,15 @@ class GoogleSTTService(STTService): ), ) + self._request_queue = asyncio.Queue() self._streaming_task = self.create_task(self._stream_audio()) async def _disconnect(self): """Clean up streaming recognition resources.""" if self._streaming_task: logger.debug("Disconnecting from Google Speech-to-Text") - # Send sentinel value to stop request generator - await self._request_queue.put(None) await self.cancel_task(self._streaming_task) self._streaming_task = None - # Clear any remaining items in the queue - while not self._request_queue.empty(): - try: - self._request_queue.get_nowait() - self._request_queue.task_done() - except asyncio.QueueEmpty: - break async def _request_generator(self): """Generates requests for the streaming recognize method.""" @@ -714,29 +751,23 @@ class GoogleSTTService(STTService): ) while True: - try: - audio_data = await self._request_queue.get() - if audio_data is None: # Sentinel value to stop - break + audio_data = await self._request_queue.get() - # Check streaming limit - if (int(time.time() * 1000) - self._stream_start_time) > self.STREAMING_LIMIT: - logger.debug("Streaming limit reached, initiating graceful reconnection") - # Instead of immediate reconnection, we'll break and let the stream close naturally - self._last_audio_input = self._audio_input - self._audio_input = [] - self._restart_counter += 1 - # Put the current audio chunk back in the queue - await self._request_queue.put(audio_data) - break + self._request_queue.task_done() - self._audio_input.append(audio_data) - yield cloud_speech.StreamingRecognizeRequest(audio=audio_data) - - except asyncio.CancelledError: + # Check streaming limit + if (int(time.time() * 1000) - self._stream_start_time) > self.STREAMING_LIMIT: + logger.debug("Streaming limit reached, initiating graceful reconnection") + # Instead of immediate reconnection, we'll break and let the stream close naturally + self._last_audio_input = self._audio_input + self._audio_input = [] + self._restart_counter += 1 + # Put the current audio chunk back in the queue + await self._request_queue.put(audio_data) break - finally: - self._request_queue.task_done() + + self._audio_input.append(audio_data) + yield cloud_speech.StreamingRecognizeRequest(audio=audio_data) except Exception as e: logger.error(f"Error in request generator: {e}") @@ -747,8 +778,6 @@ class GoogleSTTService(STTService): try: while True: try: - self.start_watchdog() - if self._request_queue.empty(): # wait for 10ms in case we don't have audio await asyncio.sleep(0.01) @@ -763,8 +792,6 @@ class GoogleSTTService(STTService): # Process responses await self._process_responses(streaming_recognize) - self.reset_watchdog() - # If we're here, check if we need to reconnect if (int(time.time() * 1000) - self._stream_start_time) > self.STREAMING_LIMIT: logger.debug("Reconnecting stream after timeout") @@ -779,15 +806,20 @@ class GoogleSTTService(STTService): await asyncio.sleep(1) # Brief delay before reconnecting self._stream_start_time = int(time.time() * 1000) - finally: - self.reset_watchdog() except Exception as e: logger.error(f"Error in streaming task: {e}") await self.push_frame(ErrorFrame(str(e))) async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Process an audio chunk for STT transcription.""" + """Process an audio chunk for STT transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: None (actual transcription frames are pushed via internal processing). + """ if self._streaming_task: # Queue the audio data await self.start_ttfb_metrics() @@ -804,17 +836,15 @@ class GoogleSTTService(STTService): async def _process_responses(self, streaming_recognize): """Process streaming recognition responses.""" try: - async for response in streaming_recognize: - self.start_watchdog() - + async for response in WatchdogAsyncIterator( + streaming_recognize, manager=self.task_manager + ): # Check streaming limit if (int(time.time() * 1000) - self._stream_start_time) > self.STREAMING_LIMIT: logger.debug("Stream timeout reached in response processing") - self.reset_watchdog() break if not response.results: - self.reset_watchdog() continue for result in response.results: @@ -856,11 +886,8 @@ class GoogleSTTService(STTService): result=result, ) ) - - self.reset_watchdog() except Exception as e: logger.error(f"Error processing Google STT responses: {e}") - self.reset_watchdog() # Re-raise the exception to let it propagate (e.g. in the case of a # timeout, propagate to _stream_audio to reconnect) raise diff --git a/src/pipecat/services/google/tts.py b/src/pipecat/services/google/tts.py index 6e57b7b8d..40de8afff 100644 --- a/src/pipecat/services/google/tts.py +++ b/src/pipecat/services/google/tts.py @@ -4,7 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio +"""Google Cloud Text-to-Speech service implementations. + +This module provides integration with Google Cloud Text-to-Speech API, +offering both HTTP-based synthesis with SSML support and streaming synthesis +for real-time applications. +""" + import json import os @@ -43,6 +49,14 @@ except ModuleNotFoundError as e: def language_to_google_tts_language(language: Language) -> Optional[str]: + """Convert a Language enum to Google TTS language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Google TTS language code, or None if not supported. + """ language_map = { # Afrikaans Language.AF: "af-ZA", @@ -203,7 +217,32 @@ def language_to_google_tts_language(language: Language) -> Optional[str]: class GoogleHttpTTSService(TTSService): + """Google Cloud Text-to-Speech HTTP service with SSML support. + + Provides text-to-speech synthesis using Google Cloud's HTTP API with + comprehensive SSML support for voice customization, prosody control, + and styling options. Ideal for applications requiring fine-grained + control over speech output. + + Note: + Requires Google Cloud credentials via service account JSON, credentials file, + or default application credentials (GOOGLE_APPLICATION_CREDENTIALS). + Chirp and Journey voices don't support SSML and will use plain text input. + """ + class InputParams(BaseModel): + """Input parameters for Google HTTP TTS voice customization. + + Parameters: + pitch: Voice pitch adjustment (e.g., "+2st", "-50%"). + rate: Speaking rate adjustment (e.g., "slow", "fast", "125%"). + volume: Volume adjustment (e.g., "loud", "soft", "+6dB"). + emphasis: Emphasis level for the text. + language: Language for synthesis. Defaults to English. + gender: Voice gender preference. + google_style: Google-specific voice style. + """ + pitch: Optional[str] = None rate: Optional[str] = None volume: Optional[str] = None @@ -222,6 +261,16 @@ class GoogleHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initializes the Google HTTP TTS service. + + Args: + credentials: JSON string containing Google Cloud service account credentials. + credentials_path: Path to Google Cloud service account JSON file. + voice_id: Google TTS voice identifier (e.g., "en-US-Standard-A"). + sample_rate: Audio sample rate in Hz. If None, uses default. + params: Voice customization parameters including pitch, rate, volume, etc. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or GoogleHttpTTSService.InputParams() @@ -245,11 +294,20 @@ class GoogleHttpTTSService(TTSService): def _create_client( self, credentials: Optional[str], credentials_path: Optional[str] ) -> texttospeech_v1.TextToSpeechAsyncClient: + """Create authenticated Google Text-to-Speech client. + + Args: + credentials: JSON string with service account credentials. + credentials_path: Path to service account JSON file. + + Returns: + Authenticated TextToSpeechAsyncClient instance. + + Raises: + ValueError: If no valid credentials are provided. + """ creds: Optional[service_account.Credentials] = None - # Create a Google Cloud service account for the Cloud Text-to-Speech API - # Using either the provided credentials JSON string or the path to a service account JSON - # file, create a Google Cloud service account and use it to authenticate with the API. if credentials: # Use provided credentials JSON string json_account_info = json.loads(credentials) @@ -271,9 +329,22 @@ class GoogleHttpTTSService(TTSService): return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Google HTTP TTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Google TTS language format. + + Args: + language: The language to convert. + + Returns: + The Google TTS-specific language code, or None if not supported. + """ return language_to_google_tts_language(language) def _construct_ssml(self, text: str) -> str: @@ -324,6 +395,14 @@ class GoogleHttpTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Google's HTTP TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -381,19 +460,13 @@ class GoogleHttpTTSService(TTSService): class GoogleTTSService(TTSService): - """Text-to-Speech service using Google Cloud Text-to-Speech API. + """Google Cloud Text-to-Speech streaming service. - Converts text to speech using Google's TTS models with streaming synthesis - for low latency. Supports multiple languages and voices. + Provides real-time text-to-speech synthesis using Google Cloud's streaming API + for low-latency applications. Optimized for Chirp 3 HD and Journey voices + with continuous audio streaming capabilities. - Args: - credentials: JSON string containing Google Cloud service account credentials. - credentials_path: Path to Google Cloud service account JSON file. - voice_id: Google TTS voice identifier (e.g., "en-US-Chirp3-HD-Charon"). - sample_rate: Audio sample rate in Hz. - params: Language only. - - Notes: + Note: Requires Google Cloud credentials via service account JSON, file path, or default application credentials (GOOGLE_APPLICATION_CREDENTIALS env var). Only Chirp 3 HD and Journey voices are supported. Use GoogleHttpTTSService for other voices. @@ -411,6 +484,12 @@ class GoogleTTSService(TTSService): """ class InputParams(BaseModel): + """Input parameters for Google streaming TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + """ + language: Optional[Language] = Language.EN def __init__( @@ -423,6 +502,16 @@ class GoogleTTSService(TTSService): params: InputParams = InputParams(), **kwargs, ): + """Initializes the Google streaming TTS service. + + Args: + credentials: JSON string containing Google Cloud service account credentials. + credentials_path: Path to Google Cloud service account JSON file. + voice_id: Google TTS voice identifier (e.g., "en-US-Chirp3-HD-Charon"). + sample_rate: Audio sample rate in Hz. If None, uses default. + params: Language configuration parameters. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or GoogleTTSService.InputParams() @@ -466,13 +555,34 @@ class GoogleTTSService(TTSService): return texttospeech_v1.TextToSpeechAsyncClient(credentials=creds) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Google streaming TTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Google TTS language format. + + Args: + language: The language to convert. + + Returns: + The Google TTS-specific language code, or None if not supported. + """ return language_to_google_tts_language(language) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate streaming speech from text using Google's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech as it's generated. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/grok/llm.py b/src/pipecat/services/grok/llm.py index a57434986..2a9704008 100644 --- a/src/pipecat/services/grok/llm.py +++ b/src/pipecat/services/grok/llm.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Grok LLM service implementation using OpenAI-compatible interface. + +This module provides a service for interacting with Grok's API through an +OpenAI-compatible interface, including specialized token usage tracking +and context aggregation functionality. +""" + from dataclasses import dataclass from loguru import logger @@ -23,13 +30,33 @@ from pipecat.services.openai.llm import ( @dataclass class GrokContextAggregatorPair: + """Pair of context aggregators for user and assistant interactions. + + Provides a convenient container for managing both user and assistant + context aggregators together for Grok LLM interactions. + + Parameters: + _user: The user context aggregator instance. + _assistant: The assistant context aggregator instance. + """ + _user: OpenAIUserContextAggregator _assistant: OpenAIAssistantContextAggregator def user(self) -> OpenAIUserContextAggregator: + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ return self._user def assistant(self) -> OpenAIAssistantContextAggregator: + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ return self._assistant @@ -38,12 +65,8 @@ class GrokLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Grok's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing Grok's API - base_url (str, optional): The base URL for Grok API. Defaults to "https://api.x.ai/v1" - model (str, optional): The model identifier to use. Defaults to "grok-3-beta" - **kwargs: Additional keyword arguments passed to OpenAILLMService + Includes specialized token usage tracking that accumulates metrics during + processing and reports final totals. """ def __init__( @@ -54,6 +77,14 @@ class GrokLLMService(OpenAILLMService): model: str = "grok-3-beta", **kwargs, ): + """Initialize the GrokLLMService with API key and model. + + Args: + api_key: The API key for accessing Grok's API. + base_url: The base URL for Grok API. Defaults to "https://api.x.ai/v1". + model: The model identifier to use. Defaults to "grok-3-beta". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) # Initialize counters for token usage metrics self._prompt_tokens = 0 @@ -63,7 +94,16 @@ class GrokLLMService(OpenAILLMService): self._is_processing = False def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for Grok API endpoint.""" + """Create OpenAI-compatible client for Grok API endpoint. + + Args: + api_key: The API key to use. If None, uses instance default. + base_url: The base URL to use. If None, uses instance default. + **kwargs: Additional arguments passed to client creation. + + Returns: + The configured client instance for Grok API. + """ logger.debug(f"Creating Grok client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) @@ -75,8 +115,8 @@ class GrokLLMService(OpenAILLMService): them once at the end of processing. Args: - context (OpenAILLMContext): The context to process, containing messages - and other information needed for the LLM interaction. + context: The context to process, containing messages and other + information needed for the LLM interaction. """ # Reset all counters and flags at the start of processing self._prompt_tokens = 0 @@ -107,8 +147,8 @@ class GrokLLMService(OpenAILLMService): The final accumulated totals are reported at the end of processing. Args: - tokens (LLMTokenUsage): The token usage metrics for the current chunk - of processing, containing prompt_tokens and completion_tokens counts. + tokens: The token usage metrics for the current chunk of processing, + containing prompt_tokens and completion_tokens counts. """ # Only accumulate metrics during active processing if not self._is_processing: @@ -130,22 +170,20 @@ class GrokLLMService(OpenAILLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> GrokContextAggregatorPair: - """Create an instance of GrokContextAggregatorPair from an - OpenAILLMContext. Constructor keyword arguments for both the user and - assistant aggregators can be provided. + """Create an instance of GrokContextAggregatorPair from an OpenAILLMContext. + + Constructor keyword arguments for both the user and assistant aggregators + can be provided. Args: - context (OpenAILLMContext): The LLM context. - user_params (LLMUserAggregatorParams, optional): User aggregator - parameters. - assistant_params (LLMAssistantAggregatorParams, optional): User - aggregator parameters. + context: The LLM context to create aggregators for. + user_params: Parameters for configuring the user aggregator. + assistant_params: Parameters for configuring the assistant aggregator. Returns: GrokContextAggregatorPair: A pair of context aggregators, one for the user and one for the assistant, encapsulated in an GrokContextAggregatorPair. - """ context.set_llm_adapter(self.get_llm_adapter()) diff --git a/src/pipecat/services/groq/llm.py b/src/pipecat/services/groq/llm.py index be2ed5e72..57f2a533d 100644 --- a/src/pipecat/services/groq/llm.py +++ b/src/pipecat/services/groq/llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Groq LLM Service implementation using OpenAI-compatible interface.""" + from loguru import logger from pipecat.services.openai.llm import OpenAILLMService @@ -14,12 +16,6 @@ class GroqLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Groq's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing Groq's API - base_url (str, optional): The base URL for Groq API. Defaults to "https://api.groq.com/openai/v1" - model (str, optional): The model identifier to use. Defaults to "llama-3.3-70b-versatile" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -30,9 +26,26 @@ class GroqLLMService(OpenAILLMService): model: str = "llama-3.3-70b-versatile", **kwargs, ): + """Initialize Groq LLM service. + + Args: + api_key: The API key for accessing Groq's API. + base_url: The base URL for Groq API. Defaults to "https://api.groq.com/openai/v1". + model: The model identifier to use. Defaults to "llama-3.3-70b-versatile". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for Groq API endpoint.""" + """Create OpenAI-compatible client for Groq API endpoint. + + Args: + api_key: API key for authentication. If None, uses instance api_key. + base_url: Base URL for the API. If None, uses instance base_url. + **kwargs: Additional arguments passed to the client constructor. + + Returns: + An OpenAI-compatible client configured for Groq's API. + """ logger.debug(f"Creating Groq client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) diff --git a/src/pipecat/services/groq/stt.py b/src/pipecat/services/groq/stt.py index 5852bedfd..1267dd85f 100644 --- a/src/pipecat/services/groq/stt.py +++ b/src/pipecat/services/groq/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Groq speech-to-text service implementation using Whisper models.""" + from typing import Optional from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -15,15 +17,6 @@ class GroqSTTService(BaseWhisperSTTService): Uses Groq's Whisper API to convert audio to text. Requires a Groq API key set via the api_key parameter or GROQ_API_KEY environment variable. - - Args: - model: Whisper model to use. Defaults to "whisper-large-v3-turbo". - api_key: Groq API key. Defaults to None. - base_url: API base URL. Defaults to "https://api.groq.com/openai/v1". - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to BaseWhisperSTTService. """ def __init__( @@ -37,6 +30,17 @@ class GroqSTTService(BaseWhisperSTTService): temperature: Optional[float] = None, **kwargs, ): + """Initialize Groq STT service. + + Args: + model: Whisper model to use. Defaults to "whisper-large-v3-turbo". + api_key: Groq API key. Defaults to None. + base_url: API base URL. Defaults to "https://api.groq.com/openai/v1". + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to BaseWhisperSTTService. + """ super().__init__( model=model, api_key=api_key, diff --git a/src/pipecat/services/groq/tts.py b/src/pipecat/services/groq/tts.py index 33fd3ce34..68ba4a598 100644 --- a/src/pipecat/services/groq/tts.py +++ b/src/pipecat/services/groq/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Groq text-to-speech service implementation.""" + import io import wave from typing import AsyncGenerator, Optional @@ -25,7 +27,21 @@ except ModuleNotFoundError as e: class GroqTTSService(TTSService): + """Groq text-to-speech service implementation. + + Provides text-to-speech synthesis using Groq's TTS API. The service + operates at a fixed 48kHz sample rate and supports various voices + and output formats. + """ + class InputParams(BaseModel): + """Input parameters for Groq TTS configuration. + + Parameters: + language: Language for speech synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -42,6 +58,17 @@ class GroqTTSService(TTSService): sample_rate: Optional[int] = GROQ_SAMPLE_RATE, **kwargs, ): + """Initialize Groq TTS service. + + Args: + api_key: Groq API key for authentication. + output_format: Audio output format. Defaults to "wav". + params: Additional input parameters for voice customization. + model_name: TTS model to use. Defaults to "playai-tts". + voice_id: Voice identifier to use. Defaults to "Celeste-PlayAI". + sample_rate: Audio sample rate. Must be 48000 Hz for Groq TTS. + **kwargs: Additional arguments passed to parent TTSService class. + """ if sample_rate != self.GROQ_SAMPLE_RATE: logger.warning(f"Groq TTS only supports {self.GROQ_SAMPLE_RATE}Hz sample rate. ") @@ -71,10 +98,23 @@ class GroqTTSService(TTSService): self._client = AsyncGroq(api_key=self._api_key) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Groq TTS service supports metrics generation. + """ return True @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Groq's TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech data. + """ logger.debug(f"{self}: Generating TTS [{text}]") measuring_ttfb = True await self.start_ttfb_metrics() diff --git a/src/pipecat/services/image_service.py b/src/pipecat/services/image_service.py index 43dbd0bb5..243e917a6 100644 --- a/src/pipecat/services/image_service.py +++ b/src/pipecat/services/image_service.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Image generation service implementation. + +Provides base functionality for AI-powered image generation services that convert +text prompts into images. +""" + from abc import abstractmethod from typing import AsyncGenerator @@ -13,15 +19,48 @@ from pipecat.services.ai_service import AIService class ImageGenService(AIService): + """Base class for image generation services. + + Processes TextFrames by using their content as prompts for image generation. + Subclasses must implement the run_image_gen method to provide actual image + generation functionality using their specific AI service. + """ + def __init__(self, **kwargs): + """Initialize the image generation service. + + Args: + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) # Renders the image. Returns an Image object. @abstractmethod async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate an image from a text prompt. + + This method must be implemented by subclasses to provide actual image + generation functionality using their specific AI service. + + Args: + prompt: The text prompt to generate an image from. + + Yields: + Frame: Frames containing the generated image (typically ImageRawFrame + or URLImageRawFrame). + """ pass async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for image generation. + + TextFrames are used as prompts for image generation, while other frames + are passed through unchanged. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, TextFrame): diff --git a/src/pipecat/services/llm_service.py b/src/pipecat/services/llm_service.py index 5619cd35e..6ef85951a 100644 --- a/src/pipecat/services/llm_service.py +++ b/src/pipecat/services/llm_service.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base classes for Large Language Model services with function calling support.""" + import asyncio import inspect from dataclasses import dataclass @@ -41,23 +43,34 @@ FunctionCallHandler = Callable[["FunctionCallParams"], Awaitable[None]] # Type alias for a callback function that handles the result of an LLM function call. class FunctionCallResultCallback(Protocol): + """Protocol for function call result callbacks. + + Handles the result of an LLM function call execution. + """ + async def __call__( self, result: Any, *, properties: Optional[FunctionCallResultProperties] = None - ) -> None: ... + ) -> None: + """Call the result callback. + + Args: + result: The result of the function call. + properties: Optional properties for the result. + """ + ... @dataclass class FunctionCallParams: """Parameters for a function call. - Attributes: - function_name (str): The name of the function being called. - arguments (Mapping[str, Any]): The arguments for the function. - tool_call_id (str): A unique identifier for the function call. - llm (LLMService): The LLMService instance being used. - context (OpenAILLMContext): The LLM context. - result_callback (FunctionCallResultCallback): Callback to handle the result of the function call. - + Parameters: + function_name: The name of the function being called. + tool_call_id: A unique identifier for the function call. + arguments: The arguments for the function. + llm: The LLMService instance being used. + context: The LLM context. + result_callback: Callback to handle the result of the function call. """ function_name: str @@ -70,14 +83,14 @@ class FunctionCallParams: @dataclass class FunctionCallRegistryItem: - """Represents an entry in our function call registry. This is what the user - registers. + """Represents an entry in the function call registry. - Attributes: - function_name (Optional[str]): The name of the function. - handler (FunctionCallHandler): The handler for processing function call parameters. - cancel_on_interruption (bool): Flag indicating whether to cancel the call on interruption. + This is what the user registers when calling register_function. + Parameters: + function_name: The name of the function (None for catch-all handler). + handler: The handler for processing function call parameters. + cancel_on_interruption: Whether to cancel the call on interruption. """ function_name: Optional[str] @@ -87,16 +100,17 @@ class FunctionCallRegistryItem: @dataclass class FunctionCallRunnerItem: - """Represents an internal function call entry to our function call - runner. The runner executes function calls in order. + """Internal function call entry for the function call runner. - Attributes: - registry_name (Optional[str]): The function call name registration (could be None). - function_name (str): The name of the function. - tool_call_id (str): A unique identifier for the function call. - arguments (Mapping[str, Any]): The arguments for the function. - context (OpenAILLMContext): The LLM context. + The runner executes function calls in order. + Parameters: + registry_item: The registry item containing handler information. + function_name: The name of the function. + tool_call_id: A unique identifier for the function call. + arguments: The arguments for the function. + context: The LLM context. + run_llm: Optional flag to control LLM execution after function call. """ registry_item: FunctionCallRegistryItem @@ -108,22 +122,27 @@ class FunctionCallRunnerItem: class LLMService(AIService): - """This is the base class for all LLM services. It handles function calling - registration and execution. The class also provides event handlers. + """Base class for all LLM services. - An event to know when an LLM service completion timeout occurs: + Handles function calling registration and execution with support for both + parallel and sequential execution modes. Provides event handlers for + completion timeouts and function call lifecycle events. - @task.event_handler("on_completion_timeout") - async def on_completion_timeout(service): - ... + Event handlers: + on_completion_timeout: Called when an LLM completion timeout occurs. + on_function_calls_started: Called when function calls are received and + execution is about to start. - And an event to know that function calls have been received from the LLM - service and that we are going to start executing them: - - @task.event_handler("on_function_calls_started") - async def on_function_calls_started(service, function_calls: Sequence[FunctionCallFromLLM]): - ... + Example: + ```python + @task.event_handler("on_completion_timeout") + async def on_completion_timeout(service): + logger.warning("LLM completion timed out") + @task.event_handler("on_function_calls_started") + async def on_function_calls_started(service, function_calls): + logger.info(f"Starting {len(function_calls)} function calls") + ``` """ # OpenAILLMAdapter is used as the default adapter since it aligns with most LLM implementations. @@ -131,6 +150,13 @@ class LLMService(AIService): adapter_class: Type[BaseLLMAdapter] = OpenAILLMAdapter def __init__(self, run_in_parallel: bool = True, **kwargs): + """Initialize the LLM service. + + Args: + run_in_parallel: Whether to run function calls in parallel or sequentially. + Defaults to True. + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._run_in_parallel = run_in_parallel self._start_callbacks = {} @@ -143,6 +169,11 @@ class LLMService(AIService): self._register_event_handler("on_completion_timeout") def get_llm_adapter(self) -> BaseLLMAdapter: + """Get the LLM adapter instance. + + Returns: + The adapter instance used for LLM communication. + """ return self._adapter def create_context_aggregator( @@ -152,24 +183,57 @@ class LLMService(AIService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> Any: + """Create a context aggregator for managing LLM conversation context. + + Must be implemented by subclasses. + + Args: + context: The LLM context to create an aggregator for. + user_params: Parameters for user message aggregation. + assistant_params: Parameters for assistant message aggregation. + + Returns: + A context aggregator instance. + """ pass async def start(self, frame: StartFrame): + """Start the LLM service. + + Args: + frame: The start frame. + """ await super().start(frame) if not self._run_in_parallel: await self._create_sequential_runner_task() async def stop(self, frame: EndFrame): + """Stop the LLM service. + + Args: + frame: The end frame. + """ await super().stop(frame) if not self._run_in_parallel: await self._cancel_sequential_runner_task() async def cancel(self, frame: CancelFrame): + """Cancel the LLM service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) if not self._run_in_parallel: await self._cancel_sequential_runner_task() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process a frame. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): @@ -188,6 +252,18 @@ class LLMService(AIService): *, cancel_on_interruption: bool = True, ): + """Register a function handler for LLM function calls. + + Args: + function_name: The name of the function to handle. Use None to handle + all function calls with a catch-all handler. + handler: The function handler. Should accept a single FunctionCallParams + parameter. + start_callback: Legacy callback function (deprecated). Put initialization + code at the top of your handler instead. + cancel_on_interruption: Whether to cancel this function call when an + interruption occurs. Defaults to True. + """ # Registering a function with the function_name set to None will run # that handler for all functions self._functions[function_name] = FunctionCallRegistryItem( @@ -210,16 +286,38 @@ class LLMService(AIService): self._start_callbacks[function_name] = start_callback def unregister_function(self, function_name: Optional[str]): + """Remove a registered function handler. + + Args: + function_name: The name of the function handler to remove. + """ del self._functions[function_name] if self._start_callbacks[function_name]: del self._start_callbacks[function_name] def has_function(self, function_name: str): + """Check if a function handler is registered. + + Args: + function_name: The name of the function to check. + + Returns: + True if the function is registered or if a catch-all handler (None) + is registered. + """ if None in self._functions.keys(): return True return function_name in self._functions.keys() async def run_function_calls(self, function_calls: Sequence[FunctionCallFromLLM]): + """Execute a sequence of function calls from the LLM. + + Triggers the on_function_calls_started event and executes functions + either in parallel or sequentially based on the run_in_parallel setting. + + Args: + function_calls: The function calls to execute. + """ if len(function_calls) == 0: return @@ -257,7 +355,7 @@ class LLMService(AIService): else: await self._sequential_runner_queue.put(runner_item) - async def call_start_function(self, context: OpenAILLMContext, function_name: str): + async def _call_start_function(self, context: OpenAILLMContext, function_name: str): if function_name in self._start_callbacks.keys(): await self._start_callbacks[function_name](function_name, self, context) elif None in self._start_callbacks.keys(): @@ -272,6 +370,18 @@ class LLMService(AIService): text_content: Optional[str] = None, video_source: Optional[str] = None, ): + """Request an image from a user. + + Pushes a UserImageRequestFrame upstream to request an image from the + specified user. + + Args: + user_id: The ID of the user to request an image from. + function_name: Optional function name associated with the request. + tool_call_id: Optional tool call ID associated with the request. + text_content: Optional text content/context for the image request. + video_source: Optional video source identifier. + """ await self.push_frame( UserImageRequestFrame( user_id=user_id, @@ -316,7 +426,7 @@ class LLMService(AIService): ) # NOTE(aleix): This needs to be removed after we remove the deprecation. - await self.call_start_function(runner_item.context, runner_item.function_name) + await self._call_start_function(runner_item.context, runner_item.function_name) # Push a function call in-progress downstream. This frame will let our # assistant context aggregator know that we are in the middle of a diff --git a/src/pipecat/services/lmnt/tts.py b/src/pipecat/services/lmnt/tts.py index ca03f7a06..57a83e3e5 100644 --- a/src/pipecat/services/lmnt/tts.py +++ b/src/pipecat/services/lmnt/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""LMNT text-to-speech service implementation.""" + import json from typing import AsyncGenerator, Optional @@ -35,6 +37,14 @@ except ModuleNotFoundError as e: def language_to_lmnt_language(language: Language) -> Optional[str]: + """Convert a Language enum to LMNT language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding LMNT language code, or None if not supported. + """ BASE_LANGUAGES = { Language.DE: "de", Language.EN: "en", @@ -71,6 +81,13 @@ def language_to_lmnt_language(language: Language) -> Optional[str]: class LmntTTSService(InterruptibleTTSService): + """LMNT real-time text-to-speech service. + + Provides real-time text-to-speech synthesis using LMNT's WebSocket API. + Supports streaming audio generation with configurable voice models and + language settings. + """ + def __init__( self, *, @@ -81,6 +98,16 @@ class LmntTTSService(InterruptibleTTSService): model: str = "aurora", **kwargs, ): + """Initialize the LMNT TTS service. + + Args: + api_key: LMNT API key for authentication. + voice_id: ID of the voice to use for synthesis. + sample_rate: Audio sample rate. If None, uses default. + language: Language for synthesis. Defaults to English. + model: TTS model to use. Defaults to "aurora". + **kwargs: Additional arguments passed to parent InterruptibleTTSService. + """ super().__init__( push_stop_frames=True, pause_frame_processing=True, @@ -99,35 +126,71 @@ class LmntTTSService(InterruptibleTTSService): self._receive_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as LMNT service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to LMNT service language format. + + Args: + language: The language to convert. + + Returns: + The LMNT-specific language code, or None if not supported. + """ return language_to_lmnt_language(language) async def start(self, frame: StartFrame): + """Start the LMNT TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the LMNT TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the LMNT TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame downstream with special handling for stop conditions. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False async def _connect(self): + """Connect to LMNT WebSocket and start receive task.""" await self._connect_websocket() if self._websocket and not self._receive_task: self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): + """Disconnect from LMNT WebSocket and clean up tasks.""" if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None @@ -181,11 +244,13 @@ class LmntTTSService(InterruptibleTTSService): self._websocket = None def _get_websocket(self): + """Get the WebSocket connection if available.""" if self._websocket: return self._websocket raise Exception("Websocket not connected") async def flush_audio(self): + """Flush any pending audio synthesis.""" if not self._websocket or self._websocket.closed: return await self._get_websocket().send(json.dumps({"flush": True})) @@ -216,7 +281,14 @@ class LmntTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate TTS audio from text.""" + """Generate TTS audio from text using LMNT's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/mcp_service.py b/src/pipecat/services/mcp_service.py index a644d8f1b..31a29354b 100644 --- a/src/pipecat/services/mcp_service.py +++ b/src/pipecat/services/mcp_service.py @@ -1,5 +1,13 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""MCP (Model Context Protocol) client for integrating external tools with LLMs.""" + import json -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Tuple from loguru import logger @@ -9,9 +17,11 @@ from pipecat.utils.base_object import BaseObject try: from mcp import ClientSession, StdioServerParameters - from mcp.client.session_group import SseServerParameters + from mcp.client.session import ClientSession + from mcp.client.session_group import SseServerParameters, StreamableHttpParameters from mcp.client.sse import sse_client from mcp.client.stdio import stdio_client + from mcp.client.streamable_http import streamablehttp_client except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error("In order to use an MCP client, you need to `pip install pipecat-ai[mcp]`.") @@ -19,26 +29,57 @@ except ModuleNotFoundError as e: class MCPClient(BaseObject): + """Client for Model Context Protocol (MCP) servers. + + Enables integration with MCP servers to provide external tools and resources + to LLMs. Supports both stdio and SSE server connections with automatic tool + registration and schema conversion. + + Raises: + TypeError: If server_params is not a supported parameter type. + """ + def __init__( self, - server_params: Union[StdioServerParameters, SseServerParameters], + server_params: Tuple[StdioServerParameters, SseServerParameters, StreamableHttpParameters], **kwargs, ): + """Initialize the MCP client with server parameters. + + Args: + server_params: Server connection parameters (stdio or SSE). + **kwargs: Additional arguments passed to the parent BaseObject. + """ super().__init__(**kwargs) self._server_params = server_params self._session = ClientSession + if isinstance(server_params, StdioServerParameters): self._client = stdio_client self._register_tools = self._stdio_register_tools elif isinstance(server_params, SseServerParameters): self._client = sse_client self._register_tools = self._sse_register_tools + elif isinstance(server_params, StreamableHttpParameters): + self._client = streamablehttp_client + self._register_tools = self._streamable_http_register_tools else: raise TypeError( - f"{self} invalid argument type: `server_params` must be either StdioServerParameters or SseServerParameters." + f"{self} invalid argument type: `server_params` must be either StdioServerParameters, SseServerParameters, or StreamableHttpParameters." ) async def register_tools(self, llm) -> ToolsSchema: + """Register all available MCP tools with an LLM service. + + Connects to the MCP server, discovers available tools, converts their + schemas to Pipecat format, and registers them with the LLM service. + + Args: + llm: The Pipecat LLM service to register tools with. + + Returns: + A ToolsSchema containing all successfully registered tools. + """ tools_schema = await self._register_tools(llm) return tools_schema @@ -46,13 +87,13 @@ class MCPClient(BaseObject): self, tool_name: str, tool_schema: Dict[str, Any] ) -> FunctionSchema: """Convert an mcp tool schema to Pipecat's FunctionSchema format. + Args: tool_name: The name of the tool tool_schema: The mcp tool schema Returns: A FunctionSchema instance """ - logger.debug(f"Converting schema for tool '{tool_name}'") logger.trace(f"Original schema: {json.dumps(tool_schema, indent=2)}") @@ -71,7 +112,8 @@ class MCPClient(BaseObject): return schema async def _sse_register_tools(self, llm) -> ToolsSchema: - """Register all available mcp.run tools with the LLM service. + """Register all available mcp tools with the LLM service. + Args: llm: The Pipecat LLM service to register tools with Returns: @@ -86,16 +128,11 @@ class MCPClient(BaseObject): context: any, result_callback: any, ) -> None: - """Wrapper for mcp.run tool calls to match Pipecat's function call interface.""" + """Wrapper for mcp tool calls to match Pipecat's function call interface.""" logger.debug(f"Executing tool '{function_name}' with call ID: {tool_call_id}") logger.trace(f"Tool arguments: {json.dumps(arguments, indent=2)}") try: - async with self._client( - url=self._server_params.url, - headers=self._server_params.headers, - timeout=self._server_params.timeout, - sse_read_timeout=self._server_params.sse_read_timeout, - ) as (read, write): + async with self._client(**self._server_params.model_dump()) as (read, write): async with self._session(read, write) as session: await session.initialize() await self._call_tool(session, function_name, arguments, result_callback) @@ -106,20 +143,17 @@ class MCPClient(BaseObject): await result_callback(error_msg) logger.debug(f"SSE server parameters: {self._server_params}") + logger.debug("Starting registration of mcp tools") - async with self._client( - url=self._server_params.url, - headers=self._server_params.headers, - timeout=self._server_params.timeout, - sse_read_timeout=self._server_params.sse_read_timeout, - ) as (read, write): + async with self._client(**self._server_params.model_dump()) as (read, write): async with self._session(read, write) as session: await session.initialize() tools_schema = await self._list_tools(session, mcp_tool_wrapper, llm) return tools_schema async def _stdio_register_tools(self, llm) -> ToolsSchema: - """Register all available mcp.run tools with the LLM service. + """Register all available mcp tools with the LLM service. + Args: llm: The Pipecat LLM service to register tools with Returns: @@ -134,7 +168,7 @@ class MCPClient(BaseObject): context: any, result_callback: any, ) -> None: - """Wrapper for mcp.run tool calls to match Pipecat's function call interface.""" + """Wrapper for mcp tool calls to match Pipecat's function call interface.""" logger.debug(f"Executing tool '{function_name}' with call ID: {tool_call_id}") logger.trace(f"Tool arguments: {json.dumps(arguments, indent=2)}") try: @@ -148,7 +182,7 @@ class MCPClient(BaseObject): logger.exception("Full exception details:") await result_callback(error_msg) - logger.debug("Starting registration of mcp.run tools") + logger.debug("Starting registration of mcp tools") async with self._client(self._server_params) as streams: async with self._session(streams[0], streams[1]) as session: @@ -156,6 +190,53 @@ class MCPClient(BaseObject): tools_schema = await self._list_tools(session, mcp_tool_wrapper, llm) return tools_schema + async def _streamable_http_register_tools(self, llm) -> ToolsSchema: + """Register all available mcp tools with the LLM service using streamable HTTP. + + Args: + llm: The Pipecat LLM service to register tools with + Returns: + A ToolsSchema containing all registered tools + """ + + async def mcp_tool_wrapper( + function_name: str, + tool_call_id: str, + arguments: Dict[str, Any], + llm: any, + context: any, + result_callback: any, + ) -> None: + """Wrapper for mcp tool calls to match Pipecat's function call interface.""" + logger.debug(f"Executing tool '{function_name}' with call ID: {tool_call_id}") + logger.trace(f"Tool arguments: {json.dumps(arguments, indent=2)}") + try: + async with self._client(**self._server_params.model_dump()) as ( + read_stream, + write_stream, + _, + ): + async with self._session(read_stream, write_stream) as session: + await session.initialize() + await self._call_tool(session, function_name, arguments, result_callback) + except Exception as e: + error_msg = f"Error calling mcp tool {function_name}: {str(e)}" + logger.error(error_msg) + logger.exception("Full exception details:") + await result_callback(error_msg) + + logger.debug("Starting registration of mcp tools using streamable HTTP") + + async with self._client(**self._server_params.model_dump()) as ( + read_stream, + write_stream, + _, + ): + async with self._session(read_stream, write_stream) as session: + await session.initialize() + tools_schema = await self._list_tools(session, mcp_tool_wrapper, llm) + return tools_schema + async def _call_tool(self, session, function_name, arguments, result_callback): logger.debug(f"Calling mcp tool '{function_name}'") try: @@ -199,8 +280,7 @@ class MCPClient(BaseObject): try: # Convert the schema function_schema = self._convert_mcp_schema_to_pipecat( - tool_name, - {"description": tool.description, "input_schema": tool.inputSchema}, + tool_name, {"description": tool.description, "input_schema": tool.inputSchema} ) # Register the wrapped function diff --git a/src/pipecat/services/mem0/memory.py b/src/pipecat/services/mem0/memory.py index 8c93d72c9..32ea829c5 100644 --- a/src/pipecat/services/mem0/memory.py +++ b/src/pipecat/services/mem0/memory.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Mem0 memory service integration for Pipecat. + +This module provides a memory service that integrates with Mem0 to store +and retrieve conversational memories, enhancing LLM context with relevant +historical information. +""" + from typing import Any, Dict, List, Optional from loguru import logger @@ -31,14 +38,21 @@ class Mem0MemoryService(FrameProcessor): This service intercepts message frames in the pipeline, stores them in Mem0, and enhances context with relevant memories before passing them downstream. - - Args: - api_key (str): The API key for accessing Mem0's API - user_id (str): The user ID to associate with memories in Mem0 - params (InputParams, optional): Configuration parameters for memory retrieval + Supports both local and cloud-based Mem0 configurations. """ class InputParams(BaseModel): + """Configuration parameters for Mem0 memory service. + + Parameters: + search_limit: Maximum number of memories to retrieve per query. + search_threshold: Minimum similarity threshold for memory retrieval. + api_version: API version to use for Mem0 client operations. + system_prompt: Prefix text for memory context messages. + add_as_system_message: Whether to add memories as system messages. + position: Position to insert memory messages in context. + """ + search_limit: int = Field(default=10, ge=1) search_threshold: float = Field(default=0.1, ge=0.0, le=1.0) api_version: str = Field(default="v2") @@ -56,6 +70,19 @@ class Mem0MemoryService(FrameProcessor): run_id: Optional[str] = None, params: Optional[InputParams] = None, ): + """Initialize the Mem0 memory service. + + Args: + api_key: The API key for accessing Mem0's cloud API. + local_config: Local configuration for Mem0 client (alternative to cloud API). + user_id: The user ID to associate with memories in Mem0. + agent_id: The agent ID to associate with memories in Mem0. + run_id: The run ID to associate with memories in Mem0. + params: Configuration parameters for memory retrieval and storage. + + Raises: + ValueError: If none of user_id, agent_id, or run_id are provided. + """ # Important: Call the parent class __init__ first super().__init__() @@ -86,7 +113,7 @@ class Mem0MemoryService(FrameProcessor): """Store messages in Mem0. Args: - messages: List of message dictionaries to store + messages: List of message dictionaries to store in memory. """ try: logger.debug(f"Storing {len(messages)} messages in Mem0") @@ -110,10 +137,10 @@ class Mem0MemoryService(FrameProcessor): """Retrieve relevant memories from Mem0. Args: - query: The query to search for relevant memories + query: The query to search for relevant memories. Returns: - List of relevant memory dictionaries + List of relevant memory dictionaries matching the query. """ try: logger.debug(f"Retrieving memories for query: {query}") @@ -154,8 +181,8 @@ class Mem0MemoryService(FrameProcessor): """Enhance the LLM context with relevant memories. Args: - context: The OpenAILLMContext to enhance - query: The query to search for relevant memories + context: The OpenAILLMContext to enhance with memory information. + query: The query to search for relevant memories. """ # Skip if this is the same query we just processed if self.last_query == query: @@ -184,8 +211,8 @@ class Mem0MemoryService(FrameProcessor): """Process incoming frames, intercept context frames for memory integration. Args: - frame: The incoming frame to process - direction: The direction of frame flow in the pipeline + frame: The incoming frame to process. + direction: The direction of frame flow in the pipeline. """ await super().process_frame(frame, direction) diff --git a/src/pipecat/services/minimax/tts.py b/src/pipecat/services/minimax/tts.py index 86f954755..4faf88b1c 100644 --- a/src/pipecat/services/minimax/tts.py +++ b/src/pipecat/services/minimax/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""MiniMax text-to-speech service implementation. + +This module provides integration with MiniMax's T2A (Text-to-Audio) API +for streaming text-to-speech synthesis. +""" + import json from typing import AsyncGenerator, Optional @@ -25,6 +31,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts def language_to_minimax_language(language: Language) -> Optional[str]: + """Convert a Language enum to MiniMax language format. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding MiniMax language name, or None if not supported. + """ BASE_LANGUAGES = { Language.AR: "Arabic", Language.CS: "Czech", @@ -71,24 +85,18 @@ def language_to_minimax_language(language: Language) -> Optional[str]: class MiniMaxHttpTTSService(TTSService): """Text-to-speech service using MiniMax's T2A (Text-to-Audio) API. + Provides streaming text-to-speech synthesis using MiniMax's HTTP API + with support for various voice settings, emotions, and audio configurations. + Supports real-time audio streaming with configurable voice parameters. + Platform documentation: https://www.minimax.io/platform/document/T2A%20V2?key=66719005a427f0c8a5701643 - - Args: - api_key: MiniMax API key for authentication. - group_id: MiniMax Group ID to identify project. - model: TTS model name (default: "speech-02-turbo"). Options include - "speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo". - voice_id: Voice identifier (default: "Calm_Woman"). - aiohttp_session: aiohttp.ClientSession for API communication. - sample_rate: Output audio sample rate in Hz (default: None, set from pipeline). - params: Additional configuration parameters. """ class InputParams(BaseModel): """Configuration parameters for MiniMax TTS. - Attributes: + Parameters: language: Language for TTS generation. speed: Speech speed (range: 0.5 to 2.0). volume: Speech volume (range: 0 to 10). @@ -117,6 +125,19 @@ class MiniMaxHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the MiniMax TTS service. + + Args: + api_key: MiniMax API key for authentication. + group_id: MiniMax Group ID to identify project. + model: TTS model name. Defaults to "speech-02-turbo". Options include + "speech-02-hd", "speech-02-turbo", "speech-01-hd", "speech-01-turbo". + voice_id: Voice identifier. Defaults to "Calm_Woman". + aiohttp_session: aiohttp.ClientSession for API communication. + sample_rate: Output audio sample rate in Hz. If None, uses pipeline default. + params: Additional configuration parameters. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or MiniMaxHttpTTSService.InputParams() @@ -175,28 +196,62 @@ class MiniMaxHttpTTSService(TTSService): self._settings["english_normalization"] = params.english_normalization def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as MiniMax service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to MiniMax service language format. + + Args: + language: The language to convert. + + Returns: + The MiniMax-specific language name, or None if not supported. + """ return language_to_minimax_language(language) def set_model_name(self, model: str): - """Set the TTS model to use""" + """Set the TTS model to use. + + Args: + model: The model name to use for synthesis. + """ self._model_name = model def set_voice(self, voice: str): - """Set the voice to use""" + """Set the voice to use. + + Args: + voice: The voice identifier to use for synthesis. + """ self._voice_id = voice if "voice_setting" in self._settings: self._settings["voice_setting"]["voice_id"] = voice async def start(self, frame: StartFrame): + """Start the MiniMax TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["audio_setting"]["sample_rate"] = self.sample_rate logger.debug(f"MiniMax TTS initialized with sample rate: {self.sample_rate}") @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using MiniMax's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") headers = { diff --git a/src/pipecat/services/moondream/vision.py b/src/pipecat/services/moondream/vision.py index 6fe44a057..88be8a2a1 100644 --- a/src/pipecat/services/moondream/vision.py +++ b/src/pipecat/services/moondream/vision.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Moondream vision service implementation. + +This module provides integration with the Moondream vision-language model +for image analysis and description generation. +""" + import asyncio from typing import AsyncGenerator @@ -23,7 +29,15 @@ except ModuleNotFoundError as e: def detect_device(): - """Detects the appropriate device to run on, and return the device and dtype.""" + """Detect the appropriate device to run on. + + Detects available hardware acceleration and selects the best device + and data type for optimal performance. + + Returns: + tuple: A tuple containing (device, dtype) where device is a torch.device + and dtype is the recommended torch data type for that device. + """ try: import intel_extension_for_pytorch @@ -40,9 +54,24 @@ def detect_device(): class MoondreamService(VisionService): + """Moondream vision-language model service. + + Provides image analysis and description generation using the Moondream + vision-language model. Supports various hardware acceleration options + including CUDA, MPS, and Intel XPU. + """ + def __init__( self, *, model="vikhyatk/moondream2", revision="2024-08-26", use_cpu=False, **kwargs ): + """Initialize the Moondream service. + + Args: + model: Hugging Face model identifier for the Moondream model. + revision: Specific model revision to use. + use_cpu: Whether to force CPU usage instead of hardware acceleration. + **kwargs: Additional arguments passed to the parent VisionService. + """ super().__init__(**kwargs) self.set_model_name(model) @@ -65,6 +94,15 @@ class MoondreamService(VisionService): logger.debug("Loaded Moondream model") async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: + """Analyze an image and generate a description. + + Args: + frame: Vision frame containing the image data and optional question text. + + Yields: + Frame: TextFrame containing the generated image description, or ErrorFrame + if analysis fails. + """ if not self._model: logger.error(f"{self} error: Moondream model not available ({self.model_name})") yield ErrorFrame("Moondream model not available") @@ -73,6 +111,14 @@ class MoondreamService(VisionService): logger.debug(f"Analyzing image: {frame}") def get_image_description(frame: VisionImageRawFrame): + """Generate description for the given image frame. + + Args: + frame: Vision frame containing image data and question. + + Returns: + str: Generated description of the image. + """ image = Image.frombytes(frame.format, frame.size, frame.image) image_embeds = self._model.encode_image(image) description = self._model.answer_question( diff --git a/src/pipecat/services/neuphonic/tts.py b/src/pipecat/services/neuphonic/tts.py index bfceca50b..8f795f36c 100644 --- a/src/pipecat/services/neuphonic/tts.py +++ b/src/pipecat/services/neuphonic/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Neuphonic text-to-speech service implementations. + +This module provides WebSocket and HTTP-based integrations with Neuphonic's +text-to-speech API for real-time audio synthesis. +""" + import asyncio import base64 import json @@ -29,6 +35,7 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection from pipecat.services.tts_service import InterruptibleTTSService, TTSService from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.tracing.service_decorators import traced_tts try: @@ -41,6 +48,14 @@ except ModuleNotFoundError as e: def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: + """Convert a Language enum to Neuphonic language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Neuphonic language code, or None if not supported. + """ BASE_LANGUAGES = { Language.DE: "de", Language.EN: "en", @@ -68,7 +83,21 @@ def language_to_neuphonic_lang_code(language: Language) -> Optional[str]: class NeuphonicTTSService(InterruptibleTTSService): + """Neuphonic real-time text-to-speech service using WebSocket streaming. + + Provides real-time text-to-speech synthesis using Neuphonic's WebSocket API. + Supports interruption handling, keepalive connections, and configurable voice + parameters for high-quality speech generation. + """ + class InputParams(BaseModel): + """Input parameters for Neuphonic TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -83,6 +112,17 @@ class NeuphonicTTSService(InterruptibleTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Neuphonic TTS service. + + Args: + api_key: Neuphonic API key for authentication. + voice_id: ID of the voice to use for synthesis. + url: WebSocket URL for the Neuphonic API. + sample_rate: Audio sample rate in Hz. Defaults to 22050. + encoding: Audio encoding format. Defaults to "pcm_linear". + params: Additional input parameters for TTS configuration. + **kwargs: Additional arguments passed to parent InterruptibleTTSService. + """ super().__init__( aggregate_sentences=True, push_text_frames=False, @@ -113,12 +153,26 @@ class NeuphonicTTSService(InterruptibleTTSService): self._keepalive_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Neuphonic service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Neuphonic service language format. + + Args: + language: The language to convert. + + Returns: + The Neuphonic-specific language code, or None if not supported. + """ return language_to_neuphonic_lang_code(language) async def _update_settings(self, settings: Mapping[str, Any]): + """Update service settings and reconnect with new configuration.""" if "voice_id" in settings: self.set_voice(settings["voice_id"]) @@ -128,28 +182,56 @@ class NeuphonicTTSService(InterruptibleTTSService): logger.info(f"Switching TTS to settings: [{self._settings}]") async def start(self, frame: StartFrame): + """Start the Neuphonic TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the Neuphonic TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the Neuphonic TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def flush_audio(self): + """Flush any pending audio synthesis by sending stop command.""" if self._websocket: msg = {"text": ""} await self._websocket.send(json.dumps(msg)) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame downstream with special handling for stop conditions. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): self._started = False async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with special handling for speech control. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) # If we received a TTSSpeakFrame and the LLM response included text (it @@ -163,6 +245,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self.resume_processing_frames() async def _connect(self): + """Connect to Neuphonic WebSocket and start background tasks.""" await self._connect_websocket() if self._websocket and not self._receive_task: @@ -172,6 +255,7 @@ class NeuphonicTTSService(InterruptibleTTSService): self._keepalive_task = self.create_task(self._keepalive_task_handler()) async def _disconnect(self): + """Disconnect from Neuphonic WebSocket and clean up tasks.""" if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None @@ -183,6 +267,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._disconnect_websocket() async def _connect_websocket(self): + """Establish WebSocket connection to Neuphonic API.""" try: if self._websocket and self._websocket.open: return @@ -208,6 +293,7 @@ class NeuphonicTTSService(InterruptibleTTSService): await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): + """Close WebSocket connection and clean up state.""" try: await self.stop_all_metrics() @@ -221,7 +307,8 @@ class NeuphonicTTSService(InterruptibleTTSService): self._websocket = None async def _receive_messages(self): - async for message in self._websocket: + """Receive and process messages from Neuphonic WebSocket.""" + async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager): if isinstance(message, str): msg = json.loads(message) if msg.get("data", {}).get("audio") is not None: @@ -232,11 +319,15 @@ class NeuphonicTTSService(InterruptibleTTSService): await self.push_frame(frame) async def _keepalive_task_handler(self): + """Handle keepalive messages to maintain WebSocket connection.""" + KEEPALIVE_SLEEP = 10 if self.task_manager.task_watchdog_enabled else 3 while True: - await asyncio.sleep(10) + self.reset_watchdog() + await asyncio.sleep(KEEPALIVE_SLEEP) await self._send_text("") async def _send_text(self, text: str): + """Send text to Neuphonic WebSocket for synthesis.""" if self._websocket: msg = {"text": text} logger.debug(f"Sending text to websocket: {msg}") @@ -244,6 +335,14 @@ class NeuphonicTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Neuphonic's streaming API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"Generating TTS: [{text}]") try: @@ -271,19 +370,21 @@ class NeuphonicTTSService(InterruptibleTTSService): class NeuphonicHttpTTSService(TTSService): - """Neuphonic Text-to-Speech service using HTTP streaming. + """Neuphonic text-to-speech service using HTTP streaming. - Args: - api_key: Neuphonic API key - voice_id: ID of the voice to use - url: Base URL for the Neuphonic API (default: "https://api.neuphonic.com") - sample_rate: Sample rate for audio output (default: 22050Hz) - encoding: Audio encoding format (default: "pcm_linear") - params: Additional parameters for TTS generation including language and speed - **kwargs: Additional keyword arguments passed to the parent class + Provides text-to-speech synthesis using Neuphonic's HTTP API with server-sent + events for streaming audio delivery. Suitable for applications that prefer + HTTP-based communication over WebSocket connections. """ class InputParams(BaseModel): + """Input parameters for Neuphonic HTTP TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 @@ -298,6 +399,17 @@ class NeuphonicHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Neuphonic HTTP TTS service. + + Args: + api_key: Neuphonic API key for authentication. + voice_id: ID of the voice to use for synthesis. + url: Base URL for the Neuphonic HTTP API. + sample_rate: Audio sample rate in Hz. Defaults to 22050. + encoding: Audio encoding format. Defaults to "pcm_linear". + params: Additional input parameters for TTS configuration. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or NeuphonicHttpTTSService.InputParams() @@ -313,12 +425,38 @@ class NeuphonicHttpTTSService(TTSService): self.set_voice(voice_id) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Neuphonic HTTP service supports metrics generation. + """ return True + def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Neuphonic service language format. + + Args: + language: The language to convert. + + Returns: + The Neuphonic-specific language code, or None if not supported. + """ + return language_to_neuphonic_lang_code(language) + async def start(self, frame: StartFrame): + """Start the Neuphonic HTTP TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) async def flush_audio(self): + """Flush any pending audio synthesis. + + Note: + HTTP-based service doesn't require explicit flushing. + """ pass @traced_tts @@ -326,9 +464,10 @@ class NeuphonicHttpTTSService(TTSService): """Generate speech from text using Neuphonic streaming API. Args: - text: The text to convert to speech + text: The text to convert to speech. + Yields: - Frames containing audio data and status information + Frame: Audio frames containing the synthesized speech and status information. """ logger.debug(f"Generating TTS: [{text}]") diff --git a/src/pipecat/services/nim/llm.py b/src/pipecat/services/nim/llm.py index d57fa8d4c..052b94274 100644 --- a/src/pipecat/services/nim/llm.py +++ b/src/pipecat/services/nim/llm.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""NVIDIA NIM API service implementation. + +This module provides a service for interacting with NVIDIA's NIM (NVIDIA Inference +Microservice) API while maintaining compatibility with the OpenAI-style interface. +""" + from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai.llm import OpenAILLMService @@ -15,12 +21,6 @@ class NimLLMService(OpenAILLMService): This service extends OpenAILLMService to work with NVIDIA's NIM API while maintaining compatibility with the OpenAI-style interface. It specifically handles the difference in token usage reporting between NIM (incremental) and OpenAI (final summary). - - Args: - api_key (str): The API key for accessing NVIDIA's NIM API - base_url (str, optional): The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1" - model (str, optional): The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -31,6 +31,14 @@ class NimLLMService(OpenAILLMService): model: str = "nvidia/llama-3.1-nemotron-70b-instruct", **kwargs, ): + """Initialize the NimLLMService. + + Args: + api_key: The API key for accessing NVIDIA's NIM API. + base_url: The base URL for NIM API. Defaults to "https://integrate.api.nvidia.com/v1". + model: The model identifier to use. Defaults to "nvidia/llama-3.1-nemotron-70b-instruct". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) # Counters for accumulating token usage metrics self._prompt_tokens = 0 @@ -47,8 +55,8 @@ class NimLLMService(OpenAILLMService): them once at the end of processing. Args: - context (OpenAILLMContext): The context to process, containing messages - and other information needed for the LLM interaction. + context: The context to process, containing messages and other information + needed for the LLM interaction. """ # Reset all counters and flags at the start of processing self._prompt_tokens = 0 @@ -79,8 +87,8 @@ class NimLLMService(OpenAILLMService): The final accumulated totals are reported at the end of processing. Args: - tokens (LLMTokenUsage): The token usage metrics for the current chunk - of processing, containing prompt_tokens and completion_tokens counts. + tokens: The token usage metrics for the current chunk of processing, + containing prompt_tokens and completion_tokens counts. """ # Only accumulate metrics during active processing if not self._is_processing: diff --git a/src/pipecat/services/ollama/llm.py b/src/pipecat/services/ollama/llm.py index bd1ac0d0d..c84505704 100644 --- a/src/pipecat/services/ollama/llm.py +++ b/src/pipecat/services/ollama/llm.py @@ -4,9 +4,24 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OLLama LLM service implementation for Pipecat AI framework.""" + from pipecat.services.openai.llm import OpenAILLMService class OLLamaLLMService(OpenAILLMService): + """OLLama LLM service that provides local language model capabilities. + + This service extends OpenAILLMService to work with locally hosted OLLama models, + providing a compatible interface for running large language models locally. + """ + def __init__(self, *, model: str = "llama2", base_url: str = "http://localhost:11434/v1"): + """Initialize OLLama LLM service. + + Args: + model: The OLLama model to use. Defaults to "llama2". + base_url: The base URL for the OLLama API endpoint. + Defaults to "http://localhost:11434/v1". + """ super().__init__(model=model, base_url=base_url, api_key="ollama") diff --git a/src/pipecat/services/openai/base_llm.py b/src/pipecat/services/openai/base_llm.py index 2badfed96..f88134fa2 100644 --- a/src/pipecat/services/openai/base_llm.py +++ b/src/pipecat/services/openai/base_llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base OpenAI LLM service implementation.""" + import base64 import json from typing import Any, Dict, List, Mapping, Optional @@ -35,20 +37,34 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.tracing.service_decorators import traced_llm class BaseOpenAILLMService(LLMService): - """This is the base for all services that use the AsyncOpenAI client. + """Base class for all services that use the AsyncOpenAI client. This service consumes OpenAILLMContextFrame frames, which contain a reference - to an OpenAILLMContext frame. The OpenAILLMContext object defines the context - sent to the LLM for a completion. This includes user, assistant and system messages - as well as tool choices and the tool, which is used if requesting function - calls from the LLM. + to an OpenAILLMContext object. The context defines what is sent to the LLM for + completion, including user, assistant, and system messages, as well as tool + choices and function call configurations. """ class InputParams(BaseModel): + """Input parameters for OpenAI model configuration. + + Parameters: + frequency_penalty: Penalty for frequent tokens (-2.0 to 2.0). + presence_penalty: Penalty for new tokens (-2.0 to 2.0). + seed: Random seed for deterministic outputs. + temperature: Sampling temperature (0.0 to 2.0). + top_k: Top-k sampling parameter (currently ignored by OpenAI). + top_p: Top-p (nucleus) sampling parameter (0.0 to 1.0). + max_tokens: Maximum tokens in response (deprecated, use max_completion_tokens). + max_completion_tokens: Maximum completion tokens to generate. + extra: Additional model-specific parameters. + """ + frequency_penalty: Optional[float] = Field( default_factory=lambda: NOT_GIVEN, ge=-2.0, le=2.0 ) @@ -77,6 +93,18 @@ class BaseOpenAILLMService(LLMService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the BaseOpenAILLMService. + + Args: + model: The OpenAI model name to use (e.g., "gpt-4.1", "gpt-4o"). + api_key: OpenAI API key. If None, uses environment variable. + base_url: Custom base URL for OpenAI API. If None, uses default. + organization: OpenAI organization ID. + project: OpenAI project ID. + default_headers: Additional HTTP headers to include in requests. + params: Input parameters for model configuration and behavior. + **kwargs: Additional arguments passed to the parent LLMService. + """ super().__init__(**kwargs) params = params or BaseOpenAILLMService.InputParams() @@ -110,6 +138,19 @@ class BaseOpenAILLMService(LLMService): default_headers=None, **kwargs, ): + """Create an AsyncOpenAI client instance. + + Args: + api_key: OpenAI API key. + base_url: Custom base URL for the API. + organization: OpenAI organization ID. + project: OpenAI project ID. + default_headers: Additional HTTP headers. + **kwargs: Additional client configuration arguments. + + Returns: + Configured AsyncOpenAI client instance. + """ return AsyncOpenAI( api_key=api_key, base_url=base_url, @@ -124,11 +165,25 @@ class BaseOpenAILLMService(LLMService): ) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as OpenAI service supports metrics generation. + """ return True async def get_chat_completions( self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] ) -> AsyncStream[ChatCompletionChunk]: + """Get streaming chat completions from OpenAI API. + + Args: + context: The LLM context containing tools and configuration. + messages: List of chat completion messages to send. + + Returns: + Async stream of chat completion chunks. + """ params = { "model": self.model_name, "stream": True, @@ -192,7 +247,7 @@ class BaseOpenAILLMService(LLMService): context ) - async for chunk in chunk_stream: + async for chunk in WatchdogAsyncIterator(chunk_stream, manager=self.task_manager): if chunk.usage: tokens = LLMTokenUsage( prompt_tokens=chunk.usage.prompt_tokens, @@ -274,6 +329,15 @@ class BaseOpenAILLMService(LLMService): await self.run_function_calls(function_calls) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for LLM completion requests. + + Handles OpenAILLMContextFrame, LLMMessagesFrame, VisionImageRawFrame, + and LLMUpdateSettingsFrame to trigger LLM completions and manage settings. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) context = None diff --git a/src/pipecat/services/openai/image.py b/src/pipecat/services/openai/image.py index 3d7f6cb70..76ec14902 100644 --- a/src/pipecat/services/openai/image.py +++ b/src/pipecat/services/openai/image.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI image generation service implementation. + +This module provides integration with OpenAI's DALL-E image generation API +for creating images from text prompts. +""" + import io from typing import AsyncGenerator, Literal, Optional @@ -21,6 +27,13 @@ from pipecat.services.image_service import ImageGenService class OpenAIImageGenService(ImageGenService): + """OpenAI DALL-E image generation service. + + Provides image generation capabilities using OpenAI's DALL-E models. + Supports various image sizes and can generate images from text prompts + with configurable quality and style parameters. + """ + def __init__( self, *, @@ -30,6 +43,15 @@ class OpenAIImageGenService(ImageGenService): image_size: Literal["256x256", "512x512", "1024x1024", "1792x1024", "1024x1792"], model: str = "dall-e-3", ): + """Initialize the OpenAI image generation service. + + Args: + api_key: OpenAI API key for authentication. + base_url: Custom base URL for OpenAI API. If None, uses default. + aiohttp_session: HTTP session for downloading generated images. + image_size: Target size for generated images. + model: DALL-E model to use for generation. Defaults to "dall-e-3". + """ super().__init__() self.set_model_name(model) self._image_size = image_size @@ -37,6 +59,14 @@ class OpenAIImageGenService(ImageGenService): self._aiohttp_session = aiohttp_session async def run_image_gen(self, prompt: str) -> AsyncGenerator[Frame, None]: + """Generate an image from a text prompt using OpenAI's DALL-E. + + Args: + prompt: Text description of the image to generate. + + Yields: + Frame: URLImageRawFrame containing the generated image data. + """ logger.debug(f"Generating image from prompt: {prompt}") image = await self._client.images.generate( diff --git a/src/pipecat/services/openai/llm.py b/src/pipecat/services/openai/llm.py index 44e0a40ce..7919dd159 100644 --- a/src/pipecat/services/openai/llm.py +++ b/src/pipecat/services/openai/llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI LLM service implementation with context aggregators.""" + import json from dataclasses import dataclass from typing import Any, Optional @@ -26,17 +28,41 @@ from pipecat.services.openai.base_llm import BaseOpenAILLMService @dataclass class OpenAIContextAggregatorPair: + """Pair of OpenAI context aggregators for user and assistant messages. + + Parameters: + _user: User context aggregator for processing user messages. + _assistant: Assistant context aggregator for processing assistant messages. + """ + _user: "OpenAIUserContextAggregator" _assistant: "OpenAIAssistantContextAggregator" def user(self) -> "OpenAIUserContextAggregator": + """Get the user context aggregator. + + Returns: + The user context aggregator instance. + """ return self._user def assistant(self) -> "OpenAIAssistantContextAggregator": + """Get the assistant context aggregator. + + Returns: + The assistant context aggregator instance. + """ return self._assistant class OpenAILLMService(BaseOpenAILLMService): + """OpenAI LLM service implementation. + + Provides a complete OpenAI LLM service with context aggregation support. + Uses the BaseOpenAILLMService for core functionality and adds OpenAI-specific + context aggregator creation. + """ + def __init__( self, *, @@ -44,6 +70,13 @@ class OpenAILLMService(BaseOpenAILLMService): params: Optional[BaseOpenAILLMService.InputParams] = None, **kwargs, ): + """Initialize OpenAI LLM service. + + Args: + model: The OpenAI model name to use. Defaults to "gpt-4.1". + params: Input parameters for model configuration. + **kwargs: Additional arguments passed to the parent BaseOpenAILLMService. + """ super().__init__(model=model, params=params, **kwargs) def create_context_aggregator( @@ -53,14 +86,15 @@ class OpenAILLMService(BaseOpenAILLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> OpenAIContextAggregatorPair: - """Create an instance of OpenAIContextAggregatorPair from an - OpenAILLMContext. Constructor keyword arguments for both the user and - assistant aggregators can be provided. + """Create OpenAI-specific context aggregators. + + Creates a pair of context aggregators optimized for OpenAI's message format, + including support for function calls, tool usage, and image handling. Args: - context (OpenAILLMContext): The LLM context. - user_params (LLMUserAggregatorParams, optional): User aggregator parameters. - assistant_params (LLMAssistantAggregatorParams, optional): User aggregator parameters. + context: The LLM context to create aggregators for. + user_params: Parameters for user message aggregation. + assistant_params: Parameters for assistant message aggregation. Returns: OpenAIContextAggregatorPair: A pair of context aggregators, one for @@ -75,11 +109,32 @@ class OpenAILLMService(BaseOpenAILLMService): class OpenAIUserContextAggregator(LLMUserContextAggregator): + """OpenAI-specific user context aggregator. + + Handles aggregation of user messages for OpenAI LLM services. + Inherits all functionality from the base LLMUserContextAggregator. + """ + pass class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): + """OpenAI-specific assistant context aggregator. + + Handles aggregation of assistant messages for OpenAI LLM services, + with specialized support for OpenAI's function calling format, + tool usage tracking, and image message handling. + """ + async def handle_function_call_in_progress(self, frame: FunctionCallInProgressFrame): + """Handle a function call in progress. + + Adds the function call to the context with an IN_PROGRESS status + to track ongoing function execution. + + Args: + frame: Frame containing function call progress information. + """ self._context.add_message( { "role": "assistant", @@ -104,6 +159,14 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle the result of a function call. + + Updates the context with the function call result, replacing any + previous IN_PROGRESS status. + + Args: + frame: Frame containing the function call result. + """ if frame.result: result = json.dumps(frame.result) await self._update_function_call_result(frame.function_name, frame.tool_call_id, result) @@ -113,6 +176,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): ) async def handle_function_call_cancel(self, frame: FunctionCallCancelFrame): + """Handle a cancelled function call. + + Updates the context to mark the function call as cancelled. + + Args: + frame: Frame containing the function call cancellation information. + """ await self._update_function_call_result( frame.function_name, frame.tool_call_id, "CANCELLED" ) @@ -129,6 +199,14 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): message["content"] = result async def handle_user_image_frame(self, frame: UserImageRawFrame): + """Handle a user image frame from a function call request. + + Marks the associated function call as completed and adds the image + to the context for processing. + + Args: + frame: Frame containing the user image and request context. + """ await self._update_function_call_result( frame.request.function_name, frame.request.tool_call_id, "COMPLETED" ) diff --git a/src/pipecat/services/openai/stt.py b/src/pipecat/services/openai/stt.py index 173205aa0..208c68d34 100644 --- a/src/pipecat/services/openai/stt.py +++ b/src/pipecat/services/openai/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI Speech-to-Text service implementation using OpenAI's transcription API.""" + from typing import Optional from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -15,15 +17,6 @@ class OpenAISTTService(BaseWhisperSTTService): Uses OpenAI's transcription API to convert audio to text. Requires an OpenAI API key set via the api_key parameter or OPENAI_API_KEY environment variable. - - Args: - model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe". - api_key: OpenAI API key. Defaults to None. - base_url: API base URL. Defaults to None. - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to BaseWhisperSTTService. """ def __init__( @@ -37,6 +30,17 @@ class OpenAISTTService(BaseWhisperSTTService): temperature: Optional[float] = None, **kwargs, ): + """Initialize OpenAI STT service. + + Args: + model: Model to use — either gpt-4o or Whisper. Defaults to "gpt-4o-transcribe". + api_key: OpenAI API key. Defaults to None. + base_url: API base URL. Defaults to None. + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to BaseWhisperSTTService. + """ super().__init__( model=model, api_key=api_key, diff --git a/src/pipecat/services/openai/tts.py b/src/pipecat/services/openai/tts.py index 946d5e396..6e82496a4 100644 --- a/src/pipecat/services/openai/tts.py +++ b/src/pipecat/services/openai/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI text-to-speech service implementation. + +This module provides integration with OpenAI's text-to-speech API for +generating high-quality synthetic speech from text input. +""" + from typing import AsyncGenerator, Dict, Literal, Optional from loguru import logger @@ -43,16 +49,8 @@ class OpenAITTSService(TTSService): """OpenAI Text-to-Speech service that generates audio from text. This service uses the OpenAI TTS API to generate PCM-encoded audio at 24kHz. - - Args: - api_key: OpenAI API key. Defaults to None. - voice: Voice ID to use. Defaults to "alloy". - model: TTS model to use. Defaults to "gpt-4o-mini-tts". - sample_rate: Output audio sample rate in Hz. Defaults to None. - **kwargs: Additional keyword arguments passed to TTSService. - - The service returns PCM-encoded audio at the specified sample rate. - + Supports multiple voice models and configurable parameters for high-quality + speech synthesis with streaming audio output. """ OPENAI_SAMPLE_RATE = 24000 # OpenAI TTS always outputs at 24kHz @@ -68,6 +66,17 @@ class OpenAITTSService(TTSService): instructions: Optional[str] = None, **kwargs, ): + """Initialize OpenAI TTS service. + + Args: + api_key: OpenAI API key for authentication. If None, uses environment variable. + base_url: Custom base URL for OpenAI API. If None, uses default. + voice: Voice ID to use for synthesis. Defaults to "alloy". + model: TTS model to use. Defaults to "gpt-4o-mini-tts". + sample_rate: Output audio sample rate in Hz. If None, uses OpenAI's default 24kHz. + instructions: Optional instructions to guide voice synthesis behavior. + **kwargs: Additional keyword arguments passed to TTSService. + """ if sample_rate and sample_rate != self.OPENAI_SAMPLE_RATE: logger.warning( f"OpenAI TTS only supports {self.OPENAI_SAMPLE_RATE}Hz sample rate. " @@ -81,13 +90,28 @@ class OpenAITTSService(TTSService): self._client = AsyncOpenAI(api_key=api_key, base_url=base_url) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as OpenAI TTS service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the TTS model to use. + + Args: + model: The model name to use for text-to-speech synthesis. + """ logger.info(f"Switching TTS model to: [{model}]") self.set_model_name(model) async def start(self, frame: StartFrame): + """Start the OpenAI TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) if self.sample_rate != self.OPENAI_SAMPLE_RATE: logger.warning( @@ -97,6 +121,14 @@ class OpenAITTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using OpenAI's TTS API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech data. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: await self.start_ttfb_metrics() diff --git a/src/pipecat/services/openai_realtime_beta/azure.py b/src/pipecat/services/openai_realtime_beta/azure.py index 799c5e686..cfa279840 100644 --- a/src/pipecat/services/openai_realtime_beta/azure.py +++ b/src/pipecat/services/openai_realtime_beta/azure.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Azure OpenAI Realtime Beta LLM service implementation.""" + from loguru import logger from .openai import OpenAIRealtimeBetaLLMService @@ -19,7 +21,12 @@ except ModuleNotFoundError as e: class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): - """Subclass of OpenAI Realtime API Service with adjustments for Azure's wss connection.""" + """Azure OpenAI Realtime Beta LLM service with Azure-specific authentication. + + Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, + using Azure's authentication headers and endpoint format. Provides the same + real-time audio and text communication capabilities as the base OpenAI service. + """ def __init__( self, @@ -28,15 +35,13 @@ class AzureRealtimeBetaLLMService(OpenAIRealtimeBetaLLMService): base_url: str, **kwargs, ): - """Constructor takes the same arguments as the parent class, OpenAIRealtimeBetaLLMService. + """Initialize Azure Realtime Beta LLM service. - Note that the following are required arguments: + Args: api_key: The API key for the Azure OpenAI service. - base_url: The base URL for the Azure OpenAI service. - - base_url should be set to the full Azure endpoint URL including the api-version and the deployment name. For example, - - wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment + base_url: The full Azure WebSocket endpoint URL including api-version and deployment. + Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + **kwargs: Additional arguments passed to parent OpenAIRealtimeBetaLLMService. """ super().__init__(base_url=base_url, api_key=api_key, **kwargs) self.api_key = api_key diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 85d1a5457..cb1c0a9f5 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI Realtime LLM context and aggregator implementations.""" + import copy import json @@ -30,7 +32,21 @@ from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame class OpenAIRealtimeLLMContext(OpenAILLMContext): + """OpenAI Realtime LLM context with session management and message conversion. + + Extends the standard OpenAI LLM context to support real-time session properties, + instruction management, and conversion between standard message formats and + realtime conversation items. + """ + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize the OpenAIRealtimeLLMContext. + + Args: + messages: Initial conversation messages. Defaults to None. + tools: Available function tools. Defaults to None. + **kwargs: Additional arguments passed to parent OpenAILLMContext. + """ super().__init__(messages=messages, tools=tools, **kwargs) self.__setup_local() @@ -43,6 +59,14 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext): @staticmethod def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": + """Upgrade a standard OpenAI LLM context to a realtime context. + + Args: + obj: The OpenAILLMContext instance to upgrade. + + Returns: + The upgraded OpenAIRealtimeLLMContext instance. + """ if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): obj.__class__ = OpenAIRealtimeLLMContext obj.__setup_local() @@ -52,6 +76,14 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext): # - finish implementing all frames def from_standard_message(self, message): + """Convert a standard message format to a realtime conversation item. + + Args: + message: The standard message dictionary to convert. + + Returns: + A ConversationItem instance for the realtime API. + """ if message.get("role") == "user": content = message.get("content") if isinstance(message.get("content"), list): @@ -79,6 +111,14 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext): logger.error(f"Unhandled message type in from_standard_message: {message}") def get_messages_for_initializing_history(self): + """Get conversation items for initializing the realtime session history. + + Converts the context's messages to a format suitable for the realtime API, + handling system instructions and conversation history packaging. + + Returns: + List of conversation items for session initialization. + """ # We can't load a long conversation history into the openai realtime api yet. (The API/model # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So # our general strategy until this is fixed is just to put everything into a first "user" @@ -133,6 +173,11 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext): ] def add_user_content_item_as_message(self, item): + """Add a user content item as a standard message to the context. + + Args: + item: The conversation item to add as a user message. + """ message = { "role": "user", "content": [{"type": "text", "text": item.content[0].transcript}], @@ -141,9 +186,25 @@ class OpenAIRealtimeLLMContext(OpenAILLMContext): class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): + """User context aggregator for OpenAI Realtime API. + + Handles user input frames and generates appropriate context updates + for the realtime conversation, including message updates and tool settings. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + async def process_frame( self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM ): + """Process incoming frames and handle realtime-specific frame types. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) # Parent does not push LLMMessagesUpdateFrame. This ensures that in a typical pipeline, # messages are only processed by the user context aggregator, which is generally what we want. But @@ -157,6 +218,11 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): await self.push_frame(frame, direction) async def push_aggregation(self): + """Push user input aggregation. + + Currently ignores all user input coming into the pipeline as realtime + audio input is handled directly by the service. + """ # for the moment, ignore all user input coming into the pipeline. # todo: think about whether/how to fix this to allow for text input from # upstream (transport/transcription, or other sources) @@ -164,6 +230,16 @@ class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Assistant context aggregator for OpenAI Realtime API. + + Handles assistant output frames from the realtime service, filtering + out duplicate text frames and managing function call results. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames @@ -171,10 +247,21 @@ class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator) # OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames, # so we need to ignore pushing those as well, as they're also TextFrames. async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process assistant frames, filtering out duplicate text content. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)): await super().process_frame(frame, direction) async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call result and notify the realtime service. + + Args: + frame: The function call result frame to handle. + """ await super().handle_function_call_result(frame) # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 289835dae..6a45add17 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -3,13 +3,14 @@ # # SPDX-License-Identifier: BSD 2-Clause License # -# + +"""Event models and data structures for OpenAI Realtime API communication.""" import json import uuid from typing import Any, Dict, List, Literal, Optional, Union -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field # # session properties @@ -17,13 +18,7 @@ from pydantic import BaseModel, Field class InputAudioTranscription(BaseModel): - """Configuration for audio transcription settings. - - Attributes: - model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). - language: Optional language code for transcription. - prompt: Optional transcription hint text. - """ + """Configuration for audio transcription settings.""" model: str = "gpt-4o-transcribe" language: Optional[str] @@ -35,10 +30,26 @@ class InputAudioTranscription(BaseModel): language: Optional[str] = None, prompt: Optional[str] = None, ): + """Initialize InputAudioTranscription. + + Args: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ super().__init__(model=model, language=language, prompt=prompt) class TurnDetection(BaseModel): + """Server-side voice activity detection configuration. + + Parameters: + type: Detection type, must be "server_vad". + threshold: Voice activity detection threshold (0.0-1.0). Defaults to 0.5. + prefix_padding_ms: Padding before speech starts in milliseconds. Defaults to 300. + silence_duration_ms: Silence duration to detect speech end in milliseconds. Defaults to 800. + """ + type: Optional[Literal["server_vad"]] = "server_vad" threshold: Optional[float] = 0.5 prefix_padding_ms: Optional[int] = 300 @@ -46,6 +57,15 @@ class TurnDetection(BaseModel): class SemanticTurnDetection(BaseModel): + """Semantic-based turn detection configuration. + + Parameters: + type: Detection type, must be "semantic_vad". + eagerness: Turn detection eagerness level. Can be "low", "medium", "high", or "auto". + create_response: Whether to automatically create responses on turn detection. + interrupt_response: Whether to interrupt ongoing responses on turn detection. + """ + type: Optional[Literal["semantic_vad"]] = "semantic_vad" eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None create_response: Optional[bool] = None @@ -53,10 +73,33 @@ class SemanticTurnDetection(BaseModel): class InputAudioNoiseReduction(BaseModel): + """Input audio noise reduction configuration. + + Parameters: + type: Noise reduction type for different microphone scenarios. + """ + type: Optional[Literal["near_field", "far_field"]] class SessionProperties(BaseModel): + """Configuration properties for an OpenAI Realtime session. + + Parameters: + modalities: Communication modalities to enable (text, audio, or both). + instructions: System instructions for the assistant. + voice: Voice ID for text-to-speech output. + input_audio_format: Format for input audio data. + output_audio_format: Format for output audio data. + input_audio_transcription: Configuration for input audio transcription. + input_audio_noise_reduction: Configuration for input audio noise reduction. + turn_detection: Turn detection configuration or False to disable. + tools: Available function tools for the assistant. + tool_choice: Tool usage strategy ("auto", "none", or "required"). + temperature: Sampling temperature for response generation. + max_response_output_tokens: Maximum tokens in response or "inf" for unlimited. + """ + modalities: Optional[List[Literal["text", "audio"]]] = None instructions: Optional[str] = None voice: Optional[str] = None @@ -80,6 +123,15 @@ class SessionProperties(BaseModel): class ItemContent(BaseModel): + """Content within a conversation item. + + Parameters: + type: Content type (text, audio, input_text, or input_audio). + text: Text content for text-based items. + audio: Base64-encoded audio data for audio items. + transcript: Transcribed text for audio items. + """ + type: Literal["text", "audio", "input_text", "input_audio"] text: Optional[str] = None audio: Optional[str] = None # base64-encoded audio @@ -87,6 +139,21 @@ class ItemContent(BaseModel): class ConversationItem(BaseModel): + """A conversation item in the realtime session. + + Parameters: + id: Unique identifier for the item, auto-generated if not provided. + object: Object type identifier for the realtime API. + type: Item type (message, function_call, or function_call_output). + status: Current status of the item. + role: Speaker role for message items (user, assistant, or system). + content: Content list for message items. + call_id: Function call identifier for function_call items. + name: Function name for function_call items. + arguments: Function arguments as JSON string for function_call items. + output: Function output as JSON string for function_call_output items. + """ + id: str = Field(default_factory=lambda: str(uuid.uuid4().hex)) object: Optional[Literal["realtime.item"]] = None type: Literal["message", "function_call", "function_call_output"] @@ -102,11 +169,31 @@ class ConversationItem(BaseModel): class RealtimeConversation(BaseModel): + """A realtime conversation session. + + Parameters: + id: Unique identifier for the conversation. + object: Object type identifier, always "realtime.conversation". + """ + id: str object: Literal["realtime.conversation"] class ResponseProperties(BaseModel): + """Properties for configuring assistant responses. + + Parameters: + modalities: Output modalities for the response. Defaults to ["audio", "text"]. + instructions: Specific instructions for this response. + voice: Voice ID for text-to-speech in this response. + output_audio_format: Audio format for this response. + tools: Available tools for this response. + tool_choice: Tool usage strategy for this response. + temperature: Sampling temperature for this response. + max_response_output_tokens: Maximum tokens for this response. + """ + modalities: Optional[List[Literal["text", "audio"]]] = ["audio", "text"] instructions: Optional[str] = None voice: Optional[str] = None @@ -121,6 +208,16 @@ class ResponseProperties(BaseModel): # error class # class RealtimeError(BaseModel): + """Error information from the realtime API. + + Parameters: + type: Error type identifier. + code: Specific error code. + message: Human-readable error message. + param: Parameter name that caused the error, if applicable. + event_id: Event ID associated with the error, if applicable. + """ + type: str code: Optional[str] = "" message: str @@ -134,14 +231,38 @@ class RealtimeError(BaseModel): class ClientEvent(BaseModel): + """Base class for client events sent to the realtime API. + + Parameters: + event_id: Unique identifier for the event, auto-generated if not provided. + """ + event_id: str = Field(default_factory=lambda: str(uuid.uuid4())) class SessionUpdateEvent(ClientEvent): + """Event to update session properties. + + Parameters: + type: Event type, always "session.update". + session: Updated session properties. + """ + type: Literal["session.update"] = "session.update" session: SessionProperties def model_dump(self, *args, **kwargs) -> Dict[str, Any]: + """Serialize the event to a dictionary. + + Handles special serialization for turn_detection where False becomes null. + + Args: + *args: Positional arguments passed to parent model_dump. + **kwargs: Keyword arguments passed to parent model_dump. + + Returns: + Dictionary representation of the event. + """ dump = super().model_dump(*args, **kwargs) # Handle turn_detection so that False is serialized as null @@ -153,25 +274,61 @@ class SessionUpdateEvent(ClientEvent): class InputAudioBufferAppendEvent(ClientEvent): + """Event to append audio data to the input buffer. + + Parameters: + type: Event type, always "input_audio_buffer.append". + audio: Base64-encoded audio data to append. + """ + type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" audio: str # base64-encoded audio class InputAudioBufferCommitEvent(ClientEvent): + """Event to commit the current input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.commit". + """ + type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" class InputAudioBufferClearEvent(ClientEvent): + """Event to clear the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.clear". + """ + type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" class ConversationItemCreateEvent(ClientEvent): + """Event to create a new conversation item. + + Parameters: + type: Event type, always "conversation.item.create". + previous_item_id: ID of the item to insert after, if any. + item: The conversation item to create. + """ + type: Literal["conversation.item.create"] = "conversation.item.create" previous_item_id: Optional[str] = None item: ConversationItem class ConversationItemTruncateEvent(ClientEvent): + """Event to truncate a conversation item's audio content. + + Parameters: + type: Event type, always "conversation.item.truncate". + item_id: ID of the item to truncate. + content_index: Index of the content to truncate within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + type: Literal["conversation.item.truncate"] = "conversation.item.truncate" item_id: str content_index: int @@ -179,21 +336,48 @@ class ConversationItemTruncateEvent(ClientEvent): class ConversationItemDeleteEvent(ClientEvent): + """Event to delete a conversation item. + + Parameters: + type: Event type, always "conversation.item.delete". + item_id: ID of the item to delete. + """ + type: Literal["conversation.item.delete"] = "conversation.item.delete" item_id: str class ConversationItemRetrieveEvent(ClientEvent): + """Event to retrieve a conversation item by ID. + + Parameters: + type: Event type, always "conversation.item.retrieve". + item_id: ID of the item to retrieve. + """ + type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" item_id: str class ResponseCreateEvent(ClientEvent): + """Event to create a new assistant response. + + Parameters: + type: Event type, always "response.create". + response: Optional response configuration properties. + """ + type: Literal["response.create"] = "response.create" response: Optional[ResponseProperties] = None class ResponseCancelEvent(ClientEvent): + """Event to cancel the current assistant response. + + Parameters: + type: Event type, always "response.cancel". + """ + type: Literal["response.cancel"] = "response.cancel" @@ -203,6 +387,13 @@ class ResponseCancelEvent(ClientEvent): class ServerEvent(BaseModel): + """Base class for server events received from the realtime API. + + Parameters: + event_id: Unique identifier for the event. + type: Type of the server event. + """ + model_config = ConfigDict(arbitrary_types_allowed=True) event_id: str @@ -210,27 +401,65 @@ class ServerEvent(BaseModel): class SessionCreatedEvent(ServerEvent): + """Event indicating a session has been created. + + Parameters: + type: Event type, always "session.created". + session: The created session properties. + """ + type: Literal["session.created"] session: SessionProperties class SessionUpdatedEvent(ServerEvent): + """Event indicating a session has been updated. + + Parameters: + type: Event type, always "session.updated". + session: The updated session properties. + """ + type: Literal["session.updated"] session: SessionProperties class ConversationCreated(ServerEvent): + """Event indicating a conversation has been created. + + Parameters: + type: Event type, always "conversation.created". + conversation: The created conversation. + """ + type: Literal["conversation.created"] conversation: RealtimeConversation class ConversationItemCreated(ServerEvent): + """Event indicating a conversation item has been created. + + Parameters: + type: Event type, always "conversation.item.created". + previous_item_id: ID of the previous item, if any. + item: The created conversation item. + """ + type: Literal["conversation.item.created"] previous_item_id: Optional[str] = None item: ConversationItem class ConversationItemInputAudioTranscriptionDelta(ServerEvent): + """Event containing incremental input audio transcription. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.delta". + item_id: ID of the conversation item being transcribed. + content_index: Index of the content within the item. + delta: Incremental transcription text. + """ + type: Literal["conversation.item.input_audio_transcription.delta"] item_id: str content_index: int @@ -238,6 +467,15 @@ class ConversationItemInputAudioTranscriptionDelta(ServerEvent): class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): + """Event indicating input audio transcription is complete. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.completed". + item_id: ID of the conversation item that was transcribed. + content_index: Index of the content within the item. + transcript: Complete transcription text. + """ + type: Literal["conversation.item.input_audio_transcription.completed"] item_id: str content_index: int @@ -245,6 +483,15 @@ class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): class ConversationItemInputAudioTranscriptionFailed(ServerEvent): + """Event indicating input audio transcription failed. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.failed". + item_id: ID of the conversation item that failed transcription. + content_index: Index of the content within the item. + error: Error details for the transcription failure. + """ + type: Literal["conversation.item.input_audio_transcription.failed"] item_id: str content_index: int @@ -252,6 +499,15 @@ class ConversationItemInputAudioTranscriptionFailed(ServerEvent): class ConversationItemTruncated(ServerEvent): + """Event indicating a conversation item has been truncated. + + Parameters: + type: Event type, always "conversation.item.truncated". + item_id: ID of the truncated conversation item. + content_index: Index of the content within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + type: Literal["conversation.item.truncated"] item_id: str content_index: int @@ -259,26 +515,63 @@ class ConversationItemTruncated(ServerEvent): class ConversationItemDeleted(ServerEvent): + """Event indicating a conversation item has been deleted. + + Parameters: + type: Event type, always "conversation.item.deleted". + item_id: ID of the deleted conversation item. + """ + type: Literal["conversation.item.deleted"] item_id: str class ConversationItemRetrieved(ServerEvent): + """Event containing a retrieved conversation item. + + Parameters: + type: Event type, always "conversation.item.retrieved". + item: The retrieved conversation item. + """ + type: Literal["conversation.item.retrieved"] item: ConversationItem class ResponseCreated(ServerEvent): + """Event indicating an assistant response has been created. + + Parameters: + type: Event type, always "response.created". + response: The created response object. + """ + type: Literal["response.created"] response: "Response" class ResponseDone(ServerEvent): + """Event indicating an assistant response is complete. + + Parameters: + type: Event type, always "response.done". + response: The completed response object. + """ + type: Literal["response.done"] response: "Response" class ResponseOutputItemAdded(ServerEvent): + """Event indicating an output item has been added to a response. + + Parameters: + type: Event type, always "response.output_item.added". + response_id: ID of the response. + output_index: Index of the output item. + item: The added conversation item. + """ + type: Literal["response.output_item.added"] response_id: str output_index: int @@ -286,6 +579,15 @@ class ResponseOutputItemAdded(ServerEvent): class ResponseOutputItemDone(ServerEvent): + """Event indicating an output item is complete. + + Parameters: + type: Event type, always "response.output_item.done". + response_id: ID of the response. + output_index: Index of the output item. + item: The completed conversation item. + """ + type: Literal["response.output_item.done"] response_id: str output_index: int @@ -293,6 +595,17 @@ class ResponseOutputItemDone(ServerEvent): class ResponseContentPartAdded(ServerEvent): + """Event indicating a content part has been added to a response. + + Parameters: + type: Event type, always "response.content_part.added". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The added content part. + """ + type: Literal["response.content_part.added"] response_id: str item_id: str @@ -302,6 +615,17 @@ class ResponseContentPartAdded(ServerEvent): class ResponseContentPartDone(ServerEvent): + """Event indicating a content part is complete. + + Parameters: + type: Event type, always "response.content_part.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The completed content part. + """ + type: Literal["response.content_part.done"] response_id: str item_id: str @@ -311,6 +635,17 @@ class ResponseContentPartDone(ServerEvent): class ResponseTextDelta(ServerEvent): + """Event containing incremental text from a response. + + Parameters: + type: Event type, always "response.text.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental text content. + """ + type: Literal["response.text.delta"] response_id: str item_id: str @@ -320,6 +655,17 @@ class ResponseTextDelta(ServerEvent): class ResponseTextDone(ServerEvent): + """Event indicating text content is complete. + + Parameters: + type: Event type, always "response.text.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + text: Complete text content. + """ + type: Literal["response.text.done"] response_id: str item_id: str @@ -329,6 +675,17 @@ class ResponseTextDone(ServerEvent): class ResponseAudioTranscriptDelta(ServerEvent): + """Event containing incremental audio transcript from a response. + + Parameters: + type: Event type, always "response.audio_transcript.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental transcript text. + """ + type: Literal["response.audio_transcript.delta"] response_id: str item_id: str @@ -338,6 +695,17 @@ class ResponseAudioTranscriptDelta(ServerEvent): class ResponseAudioTranscriptDone(ServerEvent): + """Event indicating audio transcript is complete. + + Parameters: + type: Event type, always "response.audio_transcript.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + transcript: Complete transcript text. + """ + type: Literal["response.audio_transcript.done"] response_id: str item_id: str @@ -347,6 +715,17 @@ class ResponseAudioTranscriptDone(ServerEvent): class ResponseAudioDelta(ServerEvent): + """Event containing incremental audio data from a response. + + Parameters: + type: Event type, always "response.audio.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Base64-encoded incremental audio data. + """ + type: Literal["response.audio.delta"] response_id: str item_id: str @@ -356,6 +735,16 @@ class ResponseAudioDelta(ServerEvent): class ResponseAudioDone(ServerEvent): + """Event indicating audio content is complete. + + Parameters: + type: Event type, always "response.audio.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + """ + type: Literal["response.audio.done"] response_id: str item_id: str @@ -364,6 +753,17 @@ class ResponseAudioDone(ServerEvent): class ResponseFunctionCallArgumentsDelta(ServerEvent): + """Event containing incremental function call arguments. + + Parameters: + type: Event type, always "response.function_call_arguments.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + delta: Incremental function arguments as JSON. + """ + type: Literal["response.function_call_arguments.delta"] response_id: str item_id: str @@ -373,6 +773,17 @@ class ResponseFunctionCallArgumentsDelta(ServerEvent): class ResponseFunctionCallArgumentsDone(ServerEvent): + """Event indicating function call arguments are complete. + + Parameters: + type: Event type, always "response.function_call_arguments.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + arguments: Complete function arguments as JSON string. + """ + type: Literal["response.function_call_arguments.done"] response_id: str item_id: str @@ -382,47 +793,111 @@ class ResponseFunctionCallArgumentsDone(ServerEvent): class InputAudioBufferSpeechStarted(ServerEvent): + """Event indicating speech has started in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_started". + audio_start_ms: Start time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + type: Literal["input_audio_buffer.speech_started"] audio_start_ms: int item_id: str class InputAudioBufferSpeechStopped(ServerEvent): + """Event indicating speech has stopped in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_stopped". + audio_end_ms: End time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + type: Literal["input_audio_buffer.speech_stopped"] audio_end_ms: int item_id: str class InputAudioBufferCommitted(ServerEvent): + """Event indicating the input audio buffer has been committed. + + Parameters: + type: Event type, always "input_audio_buffer.committed". + previous_item_id: ID of the previous item, if any. + item_id: ID of the committed conversation item. + """ + type: Literal["input_audio_buffer.committed"] previous_item_id: Optional[str] = None item_id: str class InputAudioBufferCleared(ServerEvent): + """Event indicating the input audio buffer has been cleared. + + Parameters: + type: Event type, always "input_audio_buffer.cleared". + """ + type: Literal["input_audio_buffer.cleared"] class ErrorEvent(ServerEvent): + """Event indicating an error occurred. + + Parameters: + type: Event type, always "error". + error: Error details. + """ + type: Literal["error"] error: RealtimeError class RateLimitsUpdated(ServerEvent): + """Event indicating rate limits have been updated. + + Parameters: + type: Event type, always "rate_limits.updated". + rate_limits: List of rate limit information. + """ + type: Literal["rate_limits.updated"] rate_limits: List[Dict[str, Any]] class TokenDetails(BaseModel): + """Detailed token usage information. + + Parameters: + cached_tokens: Number of cached tokens used. Defaults to 0. + text_tokens: Number of text tokens used. Defaults to 0. + audio_tokens: Number of audio tokens used. Defaults to 0. + """ + cached_tokens: Optional[int] = 0 text_tokens: Optional[int] = 0 audio_tokens: Optional[int] = 0 class Config: + """Pydantic configuration for TokenDetails.""" + extra = "allow" class Usage(BaseModel): + """Token usage statistics for a response. + + Parameters: + total_tokens: Total number of tokens used. + input_tokens: Number of input tokens used. + output_tokens: Number of output tokens used. + input_token_details: Detailed breakdown of input token usage. + output_token_details: Detailed breakdown of output token usage. + """ + total_tokens: int input_tokens: int output_tokens: int @@ -431,6 +906,17 @@ class Usage(BaseModel): class Response(BaseModel): + """A complete assistant response. + + Parameters: + id: Unique identifier for the response. + object: Object type, always "realtime.response". + status: Current status of the response. + status_details: Additional status information. + output: List of conversation items in the response. + usage: Token usage statistics for the response. + """ + id: str object: Literal["realtime.response"] status: Literal["completed", "in_progress", "incomplete", "cancelled", "failed"] @@ -474,6 +960,17 @@ _server_event_types = { def parse_server_event(str): + """Parse a server event from JSON string. + + Args: + str: JSON string containing the server event. + + Returns: + Parsed server event object of the appropriate type. + + Raises: + Exception: If the event type is unimplemented or parsing fails. + """ try: event = json.loads(str) event_type = event["type"] diff --git a/src/pipecat/services/openai_realtime_beta/frames.py b/src/pipecat/services/openai_realtime_beta/frames.py index 39de49b34..25e7c409c 100644 --- a/src/pipecat/services/openai_realtime_beta/frames.py +++ b/src/pipecat/services/openai_realtime_beta/frames.py @@ -4,16 +4,34 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Custom frame types for OpenAI Realtime API integration.""" + from dataclasses import dataclass +from typing import TYPE_CHECKING from pipecat.frames.frames import DataFrame, FunctionCallResultFrame +if TYPE_CHECKING: + from pipecat.services.openai_realtime_beta.context import OpenAIRealtimeLLMContext + @dataclass class RealtimeMessagesUpdateFrame(DataFrame): + """Frame indicating that the realtime context messages have been updated. + + Parameters: + context: The updated OpenAI realtime LLM context. + """ + context: "OpenAIRealtimeLLMContext" @dataclass class RealtimeFunctionCallResultFrame(DataFrame): + """Frame containing function call results for the realtime service. + + Parameters: + result_frame: The function call result frame to send to the realtime API. + """ + result_frame: FunctionCallResultFrame diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 7f459000a..ce21e33ad 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenAI Realtime Beta LLM service implementation with WebSocket support.""" + import base64 import json import time @@ -51,8 +53,9 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.services.llm_service import FunctionCallFromLLM, LLMService from pipecat.services.openai.llm import OpenAIContextAggregatorPair from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.time import time_now_iso8601 -from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt, traced_tts +from pipecat.utils.tracing.service_decorators import traced_openai_realtime, traced_stt from . import events from .context import ( @@ -72,6 +75,15 @@ except ModuleNotFoundError as e: @dataclass class CurrentAudioResponse: + """Tracks the current audio response from the assistant. + + Parameters: + item_id: Unique identifier for the audio response item. + content_index: Index of the audio content within the item. + start_time_ms: Timestamp when the audio response started in milliseconds. + total_size: Total size of audio data received in bytes. Defaults to 0. + """ + item_id: str content_index: int start_time_ms: int @@ -79,6 +91,13 @@ class CurrentAudioResponse: class OpenAIRealtimeBetaLLMService(LLMService): + """OpenAI Realtime Beta LLM service providing real-time audio and text communication. + + Implements the OpenAI Realtime API Beta with WebSocket communication for low-latency + bidirectional audio and text interactions. Supports function calling, conversation + management, and real-time transcription. + """ + # Overriding the default adapter to use the OpenAIRealtimeLLMAdapter one. adapter_class = OpenAIRealtimeLLMAdapter @@ -93,6 +112,19 @@ class OpenAIRealtimeBetaLLMService(LLMService): send_transcription_frames: bool = True, **kwargs, ): + """Initialize the OpenAI Realtime Beta LLM service. + + Args: + api_key: OpenAI API key for authentication. + model: OpenAI model name. Defaults to "gpt-4o-realtime-preview-2025-06-03". + base_url: WebSocket base URL for the realtime API. + Defaults to "wss://api.openai.com/v1/realtime". + session_properties: Configuration properties for the realtime session. + If None, uses default SessionProperties. + start_audio_paused: Whether to start with audio input paused. Defaults to False. + send_transcription_frames: Whether to emit transcription frames. Defaults to True. + **kwargs: Additional arguments passed to parent LLMService. + """ full_url = f"{base_url}?model={model}" super().__init__(base_url=full_url, **kwargs) @@ -124,12 +156,30 @@ class OpenAIRealtimeBetaLLMService(LLMService): self._retrieve_conversation_item_futures = {} def can_generate_metrics(self) -> bool: + """Check if the service can generate usage metrics. + + Returns: + True if metrics generation is supported. + """ return True def set_audio_input_paused(self, paused: bool): + """Set whether audio input is paused. + + Args: + paused: True to pause audio input, False to resume. + """ self._audio_input_paused = paused async def retrieve_conversation_item(self, item_id: str): + """Retrieve a conversation item by ID from the server. + + Args: + item_id: The ID of the conversation item to retrieve. + + Returns: + The retrieved conversation item. + """ future = self.get_event_loop().create_future() retrieval_in_flight = False if not self._retrieve_conversation_item_futures.get(item_id): @@ -153,14 +203,29 @@ class OpenAIRealtimeBetaLLMService(LLMService): # async def start(self, frame: StartFrame): + """Start the service and establish WebSocket connection. + + Args: + frame: The start frame triggering service initialization. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the service and close WebSocket connection. + + Args: + frame: The end frame triggering service shutdown. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the service and close WebSocket connection. + + Args: + frame: The cancel frame triggering service cancellation. + """ await super().cancel(frame) await self._disconnect() @@ -246,6 +311,12 @@ class OpenAIRealtimeBetaLLMService(LLMService): # async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames from the pipeline. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ await super().process_frame(frame, direction) if isinstance(frame, TranscriptionFrame): @@ -303,6 +374,11 @@ class OpenAIRealtimeBetaLLMService(LLMService): # async def send_client_event(self, event: events.ClientEvent): + """Send a client event to the OpenAI Realtime API. + + Args: + event: The client event to send. + """ await self._ws_send(event.model_dump(exclude_none=True)) async def _connect(self): @@ -369,8 +445,7 @@ class OpenAIRealtimeBetaLLMService(LLMService): # async def _receive_task_handler(self): - async for message in self._websocket: - self.start_watchdog() + async for message in WatchdogAsyncIterator(self._websocket, manager=self.task_manager): evt = events.parse_server_event(message) if evt.type == "session.created": await self._handle_evt_session_created(evt) @@ -401,7 +476,6 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self._handle_evt_error(evt) # errors are fatal, so exit the receive loop return - self.reset_watchdog() @traced_openai_realtime(operation="llm_setup") async def _handle_evt_session_created(self, evt): @@ -477,6 +551,11 @@ class OpenAIRealtimeBetaLLMService(LLMService): pass async def handle_evt_input_audio_transcription_completed(self, evt): + """Handle completion of input audio transcription. + + Args: + evt: The transcription completed event. + """ await self._call_event_handler("on_conversation_item_updated", evt.item_id, None) if self._send_transcription_frames: @@ -557,7 +636,9 @@ class OpenAIRealtimeBetaLLMService(LLMService): await self.push_frame(UserStoppedSpeakingFrame()) async def _maybe_handle_evt_retrieve_conversation_item_error(self, evt: events.ErrorEvent): - """If the given error event is an error retrieving a conversation item: + """Maybe handle an error event related to retrieving a conversation item. + + If the given error event is an error retrieving a conversation item: - set an exception on the future that retrieve_conversation_item() is waiting on - return true Otherwise: @@ -604,8 +685,11 @@ class OpenAIRealtimeBetaLLMService(LLMService): # async def reset_conversation(self): - # Disconnect/reconnect is the safest way to start a new conversation. - # Note that this will fail if called from the receive task. + """Reset the conversation by disconnecting and reconnecting. + + This is the safest way to start a new conversation. Note that this will + fail if called from the receive task. + """ logger.debug("Resetting conversation") await self._disconnect() if self._context: @@ -653,22 +737,19 @@ class OpenAIRealtimeBetaLLMService(LLMService): user_params: LLMUserAggregatorParams = LLMUserAggregatorParams(), assistant_params: LLMAssistantAggregatorParams = LLMAssistantAggregatorParams(), ) -> OpenAIContextAggregatorPair: - """Create an instance of OpenAIContextAggregatorPair from an - OpenAILLMContext. Constructor keyword arguments for both the user and - assistant aggregators can be provided. + """Create an instance of OpenAIContextAggregatorPair from an OpenAILLMContext. + + Constructor keyword arguments for both the user and assistant aggregators can be provided. Args: - context (OpenAILLMContext): The LLM context. - user_params (LLMUserAggregatorParams, optional): User aggregator - parameters. - assistant_params (LLMAssistantAggregatorParams, optional): User - aggregator parameters. + context: The LLM context. + user_params: User aggregator parameters. + assistant_params: Assistant aggregator parameters. Returns: OpenAIContextAggregatorPair: A pair of context aggregators, one for the user and one for the assistant, encapsulated in an OpenAIContextAggregatorPair. - """ context.set_llm_adapter(self.get_llm_adapter()) diff --git a/src/pipecat/services/openpipe/llm.py b/src/pipecat/services/openpipe/llm.py index 2a2dd1d26..25257e294 100644 --- a/src/pipecat/services/openpipe/llm.py +++ b/src/pipecat/services/openpipe/llm.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenPipe LLM service implementation for Pipecat. + +This module provides an OpenPipe-specific implementation of the OpenAI LLM service, +enabling integration with OpenPipe's fine-tuning and monitoring capabilities. +""" + from typing import Dict, List, Optional from loguru import logger @@ -22,6 +28,13 @@ except ModuleNotFoundError as e: class OpenPipeLLMService(OpenAILLMService): + """OpenPipe-powered Large Language Model service. + + Extends OpenAI's LLM service to integrate with OpenPipe's fine-tuning and + monitoring platform. Provides enhanced request logging and tagging capabilities + for model training and evaluation. + """ + def __init__( self, *, @@ -33,6 +46,17 @@ class OpenPipeLLMService(OpenAILLMService): tags: Optional[Dict[str, str]] = None, **kwargs, ): + """Initialize OpenPipe LLM service. + + Args: + model: The model name to use. Defaults to "gpt-4.1". + api_key: OpenAI API key for authentication. If None, reads from environment. + base_url: Custom OpenAI API endpoint URL. Uses default if None. + openpipe_api_key: OpenPipe API key for enhanced features. If None, reads from environment. + openpipe_base_url: OpenPipe API endpoint URL. Defaults to "https://app.openpipe.ai/api/v1". + tags: Optional dictionary of tags to apply to all requests for tracking. + **kwargs: Additional arguments passed to parent OpenAILLMService. + """ super().__init__( model=model, api_key=api_key, @@ -44,6 +68,16 @@ class OpenPipeLLMService(OpenAILLMService): self._tags = tags def create_client(self, api_key=None, base_url=None, **kwargs): + """Create an OpenPipe client instance. + + Args: + api_key: OpenAI API key for authentication. + base_url: OpenAI API base URL. + **kwargs: Additional arguments including openpipe_api_key and openpipe_base_url. + + Returns: + Configured OpenPipe AsyncOpenAI client instance. + """ openpipe_api_key = kwargs.get("openpipe_api_key") or "" openpipe_base_url = kwargs.get("openpipe_base_url") or "" client = OpenPipeAI( @@ -56,6 +90,15 @@ class OpenPipeLLMService(OpenAILLMService): async def get_chat_completions( self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] ) -> AsyncStream[ChatCompletionChunk]: + """Generate streaming chat completions with OpenPipe logging. + + Args: + context: The OpenAI LLM context containing conversation state. + messages: List of chat completion message parameters. + + Returns: + Async stream of chat completion chunks. + """ chunks = await self._client.chat.completions.create( model=self.model_name, stream=True, diff --git a/src/pipecat/services/openrouter/llm.py b/src/pipecat/services/openrouter/llm.py index 431724f94..97a9d336a 100644 --- a/src/pipecat/services/openrouter/llm.py +++ b/src/pipecat/services/openrouter/llm.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""OpenRouter LLM service implementation. + +This module provides an OpenAI-compatible interface for interacting with OpenRouter's API, +extending the base OpenAI LLM service functionality. +""" + from typing import Optional from loguru import logger @@ -16,12 +22,6 @@ class OpenRouterLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to OpenRouter's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing OpenRouter's API - base_url (str, optional): The base URL for OpenRouter API. Defaults to "https://openrouter.ai/api/v1" - model (str, optional): The model identifier to use. Defaults to "openai/gpt-4o-2024-11-20" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -32,6 +32,15 @@ class OpenRouterLLMService(OpenAILLMService): base_url: str = "https://openrouter.ai/api/v1", **kwargs, ): + """Initialize the OpenRouter LLM service. + + Args: + api_key: The API key for accessing OpenRouter's API. If None, will attempt + to read from environment variables. + model: The model identifier to use. Defaults to "openai/gpt-4o-2024-11-20". + base_url: The base URL for OpenRouter API. Defaults to "https://openrouter.ai/api/v1". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__( api_key=api_key, base_url=base_url, @@ -40,5 +49,15 @@ class OpenRouterLLMService(OpenAILLMService): ) def create_client(self, api_key=None, base_url=None, **kwargs): + """Create an OpenRouter API client. + + Args: + api_key: The API key to use for authentication. If None, uses instance default. + base_url: The base URL for the API. If None, uses instance default. + **kwargs: Additional arguments passed to the parent client creation method. + + Returns: + The configured OpenRouter API client instance. + """ logger.debug(f"Creating OpenRouter client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) diff --git a/src/pipecat/services/perplexity/llm.py b/src/pipecat/services/perplexity/llm.py index ff9f82bdb..ae80b3942 100644 --- a/src/pipecat/services/perplexity/llm.py +++ b/src/pipecat/services/perplexity/llm.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Perplexity LLM service implementation. + +This module provides a service for interacting with Perplexity's API using +an OpenAI-compatible interface. It handles Perplexity's unique token usage +reporting patterns while maintaining compatibility with the Pipecat framework. +""" + from typing import List from openai import NOT_GIVEN, AsyncStream @@ -20,12 +27,6 @@ class PerplexityLLMService(OpenAILLMService): This service extends OpenAILLMService to work with Perplexity's API while maintaining compatibility with the OpenAI-style interface. It specifically handles the difference in token usage reporting between Perplexity (incremental) and OpenAI (final summary). - - Args: - api_key (str): The API key for accessing Perplexity's API - base_url (str, optional): The base URL for Perplexity's API. Defaults to "https://api.perplexity.ai" - model (str, optional): The model identifier to use. Defaults to "sonar" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -36,6 +37,14 @@ class PerplexityLLMService(OpenAILLMService): model: str = "sonar", **kwargs, ): + """Initialize the Perplexity LLM service. + + Args: + api_key: The API key for accessing Perplexity's API. + base_url: The base URL for Perplexity's API. Defaults to "https://api.perplexity.ai". + model: The model identifier to use. Defaults to "sonar". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) # Counters for accumulating token usage metrics self._prompt_tokens = 0 @@ -50,11 +59,11 @@ class PerplexityLLMService(OpenAILLMService): """Get chat completions from Perplexity API using OpenAI-compatible parameters. Args: - context: The context containing conversation history and settings - messages: The messages to send to the API + context: The context containing conversation history and settings. + messages: The messages to send to the API. Returns: - A stream of chat completion chunks + A stream of chat completion chunks from the Perplexity API. """ params = { "model": self.model_name, @@ -85,8 +94,8 @@ class PerplexityLLMService(OpenAILLMService): and reporting them once at the end of processing. Args: - context (OpenAILLMContext): The context to process, containing messages - and other information needed for the LLM interaction. + context: The context to process, containing messages and other + information needed for the LLM interaction. """ # Reset all counters and flags at the start of processing self._prompt_tokens = 0 @@ -115,6 +124,9 @@ class PerplexityLLMService(OpenAILLMService): Perplexity reports token usage incrementally during streaming, unlike OpenAI which provides a final summary. We accumulate the counts and report the total at the end of processing. + + Args: + tokens: Token usage information to accumulate. """ if not self._is_processing: return diff --git a/src/pipecat/services/piper/tts.py b/src/pipecat/services/piper/tts.py index 65caa3650..90176dab8 100644 --- a/src/pipecat/services/piper/tts.py +++ b/src/pipecat/services/piper/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Piper TTS service implementation.""" + from typing import AsyncGenerator, Optional import aiohttp @@ -24,12 +26,9 @@ from pipecat.utils.tracing.service_decorators import traced_tts class PiperTTSService(TTSService): """Piper TTS service implementation. - Provides integration with Piper's TTS server. - - Args: - base_url: API base URL - aiohttp_session: aiohttp ClientSession - sample_rate: Output sample rate + Provides integration with Piper's HTTP TTS server for text-to-speech + synthesis. Supports streaming audio generation with configurable sample + rates and automatic WAV header removal. """ def __init__( @@ -42,6 +41,14 @@ class PiperTTSService(TTSService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the Piper TTS service. + + Args: + base_url: Base URL for the Piper TTS HTTP server. + aiohttp_session: aiohttp ClientSession for making HTTP requests. + sample_rate: Output sample rate. If None, uses the voice model's native rate. + **kwargs: Additional arguments passed to the parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) if base_url.endswith("/"): @@ -53,17 +60,22 @@ class PiperTTSService(TTSService): self._settings = {"base_url": base_url} def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Piper service supports metrics generation. + """ return True @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text using Piper API. + """Generate speech from text using Piper's HTTP API. Args: - text: The text to convert to speech + text: The text to convert to speech. Yields: - Frames containing audio data and status information + Frame: Audio frames containing the synthesized speech and status frames. """ logger.debug(f"{self}: Generating TTS [{text}]") headers = { diff --git a/src/pipecat/services/playht/tts.py b/src/pipecat/services/playht/tts.py index 34fc6c81a..65c9fd41e 100644 --- a/src/pipecat/services/playht/tts.py +++ b/src/pipecat/services/playht/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""PlayHT text-to-speech service implementations. + +This module provides integration with PlayHT's text-to-speech API +supporting both WebSocket streaming and HTTP-based synthesis. +""" + import io import json import struct @@ -42,6 +48,14 @@ except ModuleNotFoundError as e: def language_to_playht_language(language: Language) -> Optional[str]: + """Convert a Language enum to PlayHT language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding PlayHT language code, or None if not supported. + """ BASE_LANGUAGES = { Language.AF: "afrikans", Language.AM: "amharic", @@ -96,7 +110,22 @@ def language_to_playht_language(language: Language) -> Optional[str]: class PlayHTTTSService(InterruptibleTTSService): + """PlayHT WebSocket-based text-to-speech service. + + Provides real-time text-to-speech synthesis using PlayHT's WebSocket API. + Supports streaming audio generation with configurable voice engines and + language settings. + """ + class InputParams(BaseModel): + """Input parameters for PlayHT TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + seed: Random seed for voice consistency. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 seed: Optional[int] = None @@ -113,6 +142,18 @@ class PlayHTTTSService(InterruptibleTTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the PlayHT WebSocket TTS service. + + Args: + api_key: PlayHT API key for authentication. + user_id: PlayHT user ID for authentication. + voice_url: URL of the voice to use for synthesis. + voice_engine: Voice engine to use. Defaults to "Play3.0-mini". + sample_rate: Audio sample rate. If None, uses default. + output_format: Audio output format. Defaults to "wav". + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to parent InterruptibleTTSService. + """ super().__init__( pause_frame_processing=True, sample_rate=sample_rate, @@ -140,30 +181,60 @@ class PlayHTTTSService(InterruptibleTTSService): self.set_voice(voice_url) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as PlayHT service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to PlayHT service language format. + + Args: + language: The language to convert. + + Returns: + The PlayHT-specific language code, or None if not supported. + """ return language_to_playht_language(language) async def start(self, frame: StartFrame): + """Start the PlayHT TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._connect() async def stop(self, frame: EndFrame): + """Stop the PlayHT TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): + """Cancel the PlayHT TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() async def _connect(self): + """Connect to PlayHT WebSocket and start receive task.""" await self._connect_websocket() if self._websocket and not self._receive_task: self._receive_task = self.create_task(self._receive_task_handler(self._report_error)) async def _disconnect(self): + """Disconnect from PlayHT WebSocket and clean up tasks.""" if self._receive_task: await self.cancel_task(self._receive_task) self._receive_task = None @@ -171,6 +242,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self._disconnect_websocket() async def _connect_websocket(self): + """Connect to PlayHT websocket.""" try: if self._websocket and self._websocket.open: return @@ -194,6 +266,7 @@ class PlayHTTTSService(InterruptibleTTSService): await self._call_event_handler("on_connection_error", f"{e}") async def _disconnect_websocket(self): + """Disconnect from PlayHT websocket.""" try: await self.stop_all_metrics() @@ -207,6 +280,7 @@ class PlayHTTTSService(InterruptibleTTSService): self._websocket = None async def _get_websocket_url(self): + """Retrieve WebSocket URL from PlayHT API.""" async with aiohttp.ClientSession() as session: async with session.post( "https://api.play.ht/api/v4/websocket-auth", @@ -235,16 +309,19 @@ class PlayHTTTSService(InterruptibleTTSService): raise Exception(f"Failed to get WebSocket URL: {response.status}") def _get_websocket(self): + """Get the WebSocket connection if available.""" if self._websocket: return self._websocket raise Exception("Websocket not connected") async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + """Handle interruption by stopping metrics and clearing request ID.""" await super()._handle_interruption(frame, direction) await self.stop_all_metrics() self._request_id = None async def _receive_messages(self): + """Receive messages from PlayHT websocket.""" async for message in self._get_websocket(): if isinstance(message, bytes): # Skip the WAV header message @@ -273,6 +350,14 @@ class PlayHTTTSService(InterruptibleTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using PlayHT's WebSocket API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -316,7 +401,22 @@ class PlayHTTTSService(InterruptibleTTSService): class PlayHTHttpTTSService(TTSService): + """PlayHT HTTP-based text-to-speech service. + + Provides text-to-speech synthesis using PlayHT's HTTP API for simpler, + non-streaming synthesis. Suitable for use cases where streaming is not + required and simpler integration is preferred. + """ + class InputParams(BaseModel): + """Input parameters for PlayHT HTTP TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English. + speed: Speech speed multiplier. Defaults to 1.0. + seed: Random seed for voice consistency. + """ + language: Optional[Language] = Language.EN speed: Optional[float] = 1.0 seed: Optional[int] = None @@ -333,6 +433,18 @@ class PlayHTHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the PlayHT HTTP TTS service. + + Args: + api_key: PlayHT API key for authentication. + user_id: PlayHT user ID for authentication. + voice_url: URL of the voice to use for synthesis. + voice_engine: Voice engine to use. Defaults to "Play3.0-mini". + protocol: Protocol to use ("http" or "ws"). Defaults to "http". + sample_rate: Audio sample rate. If None, uses default. + params: Additional input parameters for voice customization. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or PlayHTHttpTTSService.InputParams() @@ -369,10 +481,16 @@ class PlayHTHttpTTSService(TTSService): self.set_voice(voice_url) async def start(self, frame: StartFrame): + """Start the PlayHT HTTP TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["sample_rate"] = self.sample_rate def _create_options(self) -> TTSOptions: + """Create TTSOptions object from current settings.""" language_str = self._settings["language"] playht_language = None if language_str: @@ -392,13 +510,34 @@ class PlayHTHttpTTSService(TTSService): ) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as PlayHT HTTP service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to PlayHT service language format. + + Args: + language: The language to convert. + + Returns: + The PlayHT-specific language code, or None if not supported. + """ return language_to_playht_language(language) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate TTS audio from text using PlayHT's HTTP API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/qwen/llm.py b/src/pipecat/services/qwen/llm.py index de910a741..648cbd9e8 100644 --- a/src/pipecat/services/qwen/llm.py +++ b/src/pipecat/services/qwen/llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Qwen LLM service implementation using OpenAI-compatible interface.""" + from loguru import logger from pipecat.services.openai.llm import OpenAILLMService @@ -14,12 +16,6 @@ class QwenLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Qwen's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing Qwen's API (DashScope API key) - base_url (str, optional): Base URL for Qwen API. Defaults to "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" - model (str, optional): The model identifier to use. Defaults to "qwen-plus". - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -30,10 +26,27 @@ class QwenLLMService(OpenAILLMService): model: str = "qwen-plus", **kwargs, ): + """Initialize the Qwen LLM service. + + Args: + api_key: The API key for accessing Qwen's API (DashScope API key). + base_url: Base URL for Qwen API. Defaults to "https://dashscope-intl.aliyuncs.com/compatible-mode/v1". + model: The model identifier to use. Defaults to "qwen-plus". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) logger.info(f"Initialized Qwen LLM service with model: {model}") def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for Qwen API endpoint.""" + """Create OpenAI-compatible client for Qwen API endpoint. + + Args: + api_key: API key for authentication. If None, uses instance default. + base_url: Base URL for the API. If None, uses instance default. + **kwargs: Additional arguments passed to the parent client creation. + + Returns: + An OpenAI-compatible client configured for Qwen's API. + """ logger.debug(f"Creating Qwen client with base URL: {base_url}") return super().create_client(api_key, base_url, **kwargs) diff --git a/src/pipecat/services/rime/tts.py b/src/pipecat/services/rime/tts.py index 821eafb23..663bda28f 100644 --- a/src/pipecat/services/rime/tts.py +++ b/src/pipecat/services/rime/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Rime text-to-speech service implementations. + +This module provides both WebSocket and HTTP-based text-to-speech services +using Rime's API for streaming and batch audio synthesis. +""" + import base64 import json import uuid @@ -47,7 +53,7 @@ def language_to_rime_language(language: Language) -> str: language: The pipecat Language enum value. Returns: - str: Three-letter language code used by Rime (e.g., 'eng' for English). + Three-letter language code used by Rime (e.g., 'eng' for English). """ LANGUAGE_MAP = { Language.DE: "ger", @@ -67,7 +73,15 @@ class RimeTTSService(AudioContextWordTTSService): """ class InputParams(BaseModel): - """Configuration parameters for Rime TTS service.""" + """Configuration parameters for Rime TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + speed_alpha: Speech speed multiplier. Defaults to 1.0. + reduce_latency: Whether to reduce latency at potential quality cost. + pause_between_brackets: Whether to add pauses between bracketed content. + phonemize_between_brackets: Whether to phonemize bracketed content. + """ language: Optional[Language] = Language.EN speed_alpha: Optional[float] = 1.0 @@ -96,6 +110,8 @@ class RimeTTSService(AudioContextWordTTSService): model: Model ID to use for synthesis. sample_rate: Audio sample rate in Hz. params: Additional configuration parameters. + text_aggregator: Custom text aggregator for processing input text. + **kwargs: Additional arguments passed to parent class. """ # Initialize with parent class settings for proper frame handling super().__init__( @@ -135,14 +151,30 @@ class RimeTTSService(AudioContextWordTTSService): self._cumulative_time = 0 # Accumulates time across messages def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Rime service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> str | None: - """Convert pipecat language to Rime language code.""" + """Convert pipecat language to Rime language code. + + Args: + language: The language to convert. + + Returns: + The Rime-specific language code, or None if not supported. + """ return language_to_rime_language(language) async def set_model(self, model: str): - """Update the TTS model.""" + """Update the TTS model. + + Args: + model: The model name to use for synthesis. + """ self._model = model await super().set_model(model) @@ -159,18 +191,30 @@ class RimeTTSService(AudioContextWordTTSService): return {"operation": "eos"} async def start(self, frame: StartFrame): - """Start the service and establish websocket connection.""" + """Start the service and establish websocket connection. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["samplingRate"] = self.sample_rate await self._connect() async def stop(self, frame: EndFrame): - """Stop the service and close connection.""" + """Stop the service and close connection. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._disconnect() async def cancel(self, frame: CancelFrame): - """Cancel current operation and clean up.""" + """Cancel current operation and clean up. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._disconnect() @@ -261,6 +305,7 @@ class RimeTTSService(AudioContextWordTTSService): return word_pairs async def flush_audio(self): + """Flush any pending audio synthesis.""" if not self._context_id or not self._websocket: return @@ -310,7 +355,12 @@ class RimeTTSService(AudioContextWordTTSService): self._context_id = None async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): - """Push frame and handle end-of-turn conditions.""" + """Push frame and handle end-of-turn conditions. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ await super().push_frame(frame, direction) if isinstance(frame, (TTSStoppedFrame, StartInterruptionFrame)): if isinstance(frame, TTSStoppedFrame): @@ -318,13 +368,13 @@ class RimeTTSService(AudioContextWordTTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - """Generate speech from text. + """Generate speech from text using Rime's streaming API. Args: text: The text to convert to speech. Yields: - Frames containing audio data and timing information. + Frame: Audio frames containing the synthesized speech. """ logger.debug(f"{self}: Generating TTS [{text}]") try: @@ -354,7 +404,24 @@ class RimeTTSService(AudioContextWordTTSService): class RimeHttpTTSService(TTSService): + """Rime HTTP-based text-to-speech service. + + Provides text-to-speech synthesis using Rime's HTTP API for batch processing. + Suitable for use cases where streaming is not required. + """ + class InputParams(BaseModel): + """Configuration parameters for Rime HTTP TTS service. + + Parameters: + language: Language for synthesis. Defaults to English. + pause_between_brackets: Whether to add pauses between bracketed content. + phonemize_between_brackets: Whether to phonemize bracketed content. + inline_speed_alpha: Inline speed control markup. + speed_alpha: Speech speed multiplier. Defaults to 1.0. + reduce_latency: Whether to reduce latency at potential quality cost. + """ + language: Optional[Language] = Language.EN pause_between_brackets: Optional[bool] = False phonemize_between_brackets: Optional[bool] = False @@ -373,6 +440,17 @@ class RimeHttpTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize Rime HTTP TTS service. + + Args: + api_key: Rime API key for authentication. + voice_id: ID of the voice to use. + aiohttp_session: Shared aiohttp session for HTTP requests. + model: Model ID to use for synthesis. + sample_rate: Audio sample rate in Hz. + params: Additional configuration parameters. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RimeHttpTTSService.InputParams() @@ -396,14 +474,34 @@ class RimeHttpTTSService(TTSService): self._settings["inlineSpeedAlpha"] = params.inline_speed_alpha def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Rime HTTP service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> str | None: - """Convert pipecat language to Rime language code.""" + """Convert pipecat language to Rime language code. + + Args: + language: The language to convert. + + Returns: + The Rime-specific language code, or None if not supported. + """ return language_to_rime_language(language) @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Rime's HTTP API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") headers = { diff --git a/src/pipecat/services/riva/stt.py b/src/pipecat/services/riva/stt.py index 91c207c8b..ba8750f91 100644 --- a/src/pipecat/services/riva/stt.py +++ b/src/pipecat/services/riva/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""NVIDIA Riva Speech-to-Text service implementations for real-time and batch transcription.""" + import asyncio from typing import AsyncGenerator, List, Mapping, Optional @@ -21,6 +23,7 @@ from pipecat.frames.frames import ( ) from pipecat.services.stt_service import SegmentedSTTService, STTService from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue from pipecat.utils.time import time_now_iso8601 from pipecat.utils.tracing.service_decorators import traced_stt @@ -86,7 +89,20 @@ def language_to_riva_language(language: Language) -> Optional[str]: class RivaSTTService(STTService): + """Real-time speech-to-text service using NVIDIA Riva streaming ASR. + + Provides real-time transcription capabilities using NVIDIA's Riva ASR models + through streaming recognition. Supports interim results and continuous audio + processing for low-latency applications. + """ + class InputParams(BaseModel): + """Configuration parameters for Riva STT service. + + Parameters: + language: Target language for transcription. Defaults to EN_US. + """ + language: Optional[Language] = Language.EN_US def __init__( @@ -102,6 +118,16 @@ class RivaSTTService(STTService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Riva STT service. + + Args: + api_key: NVIDIA API key for authentication. + server: Riva server address. Defaults to NVIDIA Cloud Function endpoint. + model_function_map: Mapping containing 'function_id' and 'model_name' for the ASR model. + sample_rate: Audio sample rate in Hz. If None, uses pipeline default. + params: Additional configuration parameters for Riva. + **kwargs: Additional arguments passed to STTService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RivaSTTService.InputParams() @@ -147,9 +173,23 @@ class RivaSTTService(STTService): self._response_task = None def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + False - this service does not support metrics generation. + """ return False async def set_model(self, model: str): + """Set the ASR model for transcription. + + Args: + model: Model name to set. + + Note: + Model cannot be changed after initialization. Use model_function_map + parameter in constructor instead. + """ logger.warning(f"Cannot set model after initialization. Set model and function id like so:") example = {"function_id": "", "model_name": ""} logger.warning( @@ -157,6 +197,11 @@ class RivaSTTService(STTService): ) async def start(self, frame: StartFrame): + """Start the Riva STT service and initialize streaming configuration. + + Args: + frame: StartFrame indicating pipeline start. + """ await super().start(frame) if self._config: @@ -198,14 +243,24 @@ class RivaSTTService(STTService): self._thread_task = self.create_task(self._thread_task_handler()) if not self._response_task: - self._response_queue = asyncio.Queue() + self._response_queue = WatchdogQueue(self.task_manager) self._response_task = self.create_task(self._response_task_handler()) async def stop(self, frame: EndFrame): + """Stop the Riva STT service and clean up resources. + + Args: + frame: EndFrame indicating pipeline stop. + """ await super().stop(frame) await self._stop_tasks() async def cancel(self, frame: CancelFrame): + """Cancel the Riva STT service operation. + + Args: + frame: CancelFrame indicating operation cancellation. + """ await super().cancel(frame) await self._stop_tasks() @@ -224,13 +279,12 @@ class RivaSTTService(STTService): streaming_config=self._config, ) for response in responses: - self.start_watchdog() + self.reset_watchdog() if not response.results: continue asyncio.run_coroutine_threadsafe( self._response_queue.put(response), self.get_event_loop() ) - self.reset_watchdog() async def _thread_task_handler(self): try: @@ -285,23 +339,43 @@ class RivaSTTService(STTService): async def _response_task_handler(self): while True: response = await self._response_queue.get() - self.start_watchdog() await self._handle_response(response) - self.reset_watchdog() + self._response_queue.task_done() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Process audio data for speech-to-text transcription. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + None - transcription results are pushed to the pipeline via frames. + """ await self.start_ttfb_metrics() await self.start_processing_metrics() await self._queue.put(audio) yield None def __next__(self) -> bytes: + """Get the next audio chunk for Riva processing. + + Returns: + Audio bytes from the queue. + + Raises: + StopIteration: When the thread is no longer running. + """ if not self._thread_running: raise StopIteration future = asyncio.run_coroutine_threadsafe(self._queue.get(), self.get_event_loop()) return future.result() def __iter__(self): + """Return iterator for audio chunk processing. + + Returns: + Self as iterator. + """ return self @@ -311,17 +385,20 @@ class RivaSegmentedSTTService(SegmentedSTTService): By default, his service uses NVIDIA's Riva Canary ASR API to perform speech-to-text transcription on audio segments. It inherits from SegmentedSTTService to handle audio buffering and speech detection. - - Args: - api_key: NVIDIA API key for authentication - server: Riva server address (defaults to NVIDIA Cloud Function endpoint) - model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID - sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate - params: Additional configuration parameters for Riva - **kwargs: Additional arguments passed to SegmentedSTTService """ class InputParams(BaseModel): + """Configuration parameters for Riva segmented STT service. + + Parameters: + language: Target language for transcription. Defaults to EN_US. + profanity_filter: Whether to filter profanity from results. + automatic_punctuation: Whether to add automatic punctuation. + verbatim_transcripts: Whether to return verbatim transcripts. + boosted_lm_words: List of words to boost in language model. + boosted_lm_score: Score boost for specified words. + """ + language: Optional[Language] = Language.EN_US profanity_filter: bool = False automatic_punctuation: bool = True @@ -342,6 +419,16 @@ class RivaSegmentedSTTService(SegmentedSTTService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Riva segmented STT service. + + Args: + api_key: NVIDIA API key for authentication + server: Riva server address (defaults to NVIDIA Cloud Function endpoint) + model_function_map: Mapping of model name and its corresponding NVIDIA Cloud Function ID + sample_rate: Audio sample rate in Hz. If not provided, uses the pipeline's rate + params: Additional configuration parameters for Riva + **kwargs: Additional arguments passed to SegmentedSTTService + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RivaSegmentedSTTService.InputParams() @@ -381,7 +468,14 @@ class RivaSegmentedSTTService(SegmentedSTTService): self._settings = {"language": self._language_enum} def language_to_service_language(self, language: Language) -> Optional[str]: - """Convert pipecat Language enum to Riva's language code.""" + """Convert pipecat Language enum to Riva's language code. + + Args: + language: Language enum value. + + Returns: + Riva language code or None if not supported. + """ return language_to_riva_language(language) def _initialize_client(self): @@ -436,10 +530,23 @@ class RivaSegmentedSTTService(SegmentedSTTService): return config def can_generate_metrics(self) -> bool: - """Indicates whether this service can generate processing metrics.""" + """Check if this service can generate processing metrics. + + Returns: + True - this service supports metrics generation. + """ return True async def set_model(self, model: str): + """Set the ASR model for transcription. + + Args: + model: Model name to set. + + Note: + Model cannot be changed after initialization. Use model_function_map + parameter in constructor instead. + """ logger.warning(f"Cannot set model after initialization. Set model and function id like so:") example = {"function_id": "", "model_name": ""} logger.warning( @@ -447,13 +554,21 @@ class RivaSegmentedSTTService(SegmentedSTTService): ) async def start(self, frame: StartFrame): - """Initialize the service when the pipeline starts.""" + """Initialize the service when the pipeline starts. + + Args: + frame: StartFrame indicating pipeline start. + """ await super().start(frame) self._initialize_client() self._config = self._create_recognition_config() async def set_language(self, language: Language): - """Set the language for the STT service.""" + """Set the language for the STT service. + + Args: + language: Target language for transcription. + """ logger.info(f"Switching STT language to: [{language}]") self._language_enum = language self._language = self.language_to_service_language(language) or "en-US" @@ -540,7 +655,11 @@ class RivaSegmentedSTTService(SegmentedSTTService): class ParakeetSTTService(RivaSTTService): - """Deprecated: Use RivaSTTService instead.""" + """Deprecated speech-to-text service using NVIDIA Parakeet models. + + This class is deprecated. Use RivaSTTService instead for equivalent functionality + with Parakeet models by specifying the appropriate model_function_map. + """ def __init__( self, @@ -555,6 +674,16 @@ class ParakeetSTTService(RivaSTTService): params: Optional[RivaSTTService.InputParams] = None, # Use parent class's type **kwargs, ): + """Initialize the Parakeet STT service. + + Args: + api_key: NVIDIA API key for authentication. + server: Riva server address. Defaults to NVIDIA Cloud Function endpoint. + model_function_map: Mapping containing 'function_id' and 'model_name' for Parakeet model. + sample_rate: Audio sample rate in Hz. If None, uses pipeline default. + params: Additional configuration parameters for Riva. + **kwargs: Additional arguments passed to RivaSTTService. + """ super().__init__( api_key=api_key, server=server, diff --git a/src/pipecat/services/riva/tts.py b/src/pipecat/services/riva/tts.py index 31850ea17..b75f09db0 100644 --- a/src/pipecat/services/riva/tts.py +++ b/src/pipecat/services/riva/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""NVIDIA Riva text-to-speech service implementation. + +This module provides integration with NVIDIA Riva's TTS services through +gRPC API for high-quality speech synthesis. +""" + import asyncio import os from typing import AsyncGenerator, Mapping, Optional @@ -37,7 +43,21 @@ RIVA_TTS_TIMEOUT_SECS = 5 class RivaTTSService(TTSService): + """NVIDIA Riva text-to-speech service. + + Provides high-quality text-to-speech synthesis using NVIDIA Riva's + cloud-based TTS models. Supports multiple voices, languages, and + configurable quality settings. + """ + class InputParams(BaseModel): + """Input parameters for Riva TTS configuration. + + Parameters: + language: Language code for synthesis. Defaults to US English. + quality: Audio quality setting (0-100). Defaults to 20. + """ + language: Optional[Language] = Language.EN_US quality: Optional[int] = 20 @@ -55,6 +75,17 @@ class RivaTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the NVIDIA Riva TTS service. + + Args: + api_key: NVIDIA API key for authentication. + server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint. + voice_id: Voice model identifier. Defaults to multilingual Ray voice. + sample_rate: Audio sample rate. If None, uses service default. + model_function_map: Dictionary containing function_id and model_name for the TTS model. + params: Additional configuration parameters for TTS synthesis. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or RivaTTSService.InputParams() @@ -82,6 +113,13 @@ class RivaTTSService(TTSService): ) async def set_model(self, model: str): + """Attempt to set the TTS model. + + Note: Model cannot be changed after initialization for Riva service. + + Args: + model: The model name to set (operation not supported). + """ logger.warning(f"Cannot set model after initialization. Set model and function id like so:") example = {"function_id": "", "model_name": ""} logger.warning( @@ -90,6 +128,15 @@ class RivaTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using NVIDIA Riva TTS. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech data. + """ + def read_audio_responses(queue: asyncio.Queue): def add_response(r): asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop()) @@ -139,6 +186,12 @@ class RivaTTSService(TTSService): class FastPitchTTSService(RivaTTSService): + """Deprecated FastPitch TTS service. + + This class is deprecated. Use RivaTTSService instead for new implementations. + Provides backward compatibility for existing FastPitch TTS integrations. + """ + def __init__( self, *, @@ -153,6 +206,17 @@ class FastPitchTTSService(RivaTTSService): params: Optional[RivaTTSService.InputParams] = None, **kwargs, ): + """Initialize the deprecated FastPitch TTS service. + + Args: + api_key: NVIDIA API key for authentication. + server: gRPC server endpoint. Defaults to NVIDIA's cloud endpoint. + voice_id: Voice model identifier. Defaults to Female-1 voice. + sample_rate: Audio sample rate. If None, uses service default. + model_function_map: Dictionary containing function_id and model_name for FastPitch model. + params: Additional configuration parameters for TTS synthesis. + **kwargs: Additional arguments passed to parent RivaTTSService. + """ super().__init__( api_key=api_key, server=server, diff --git a/src/pipecat/services/sambanova/llm.py b/src/pipecat/services/sambanova/llm.py index 01a8d294c..c11489e66 100644 --- a/src/pipecat/services/sambanova/llm.py +++ b/src/pipecat/services/sambanova/llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""SambaNova LLM service implementation using OpenAI-compatible interface.""" + import json from typing import Any, Dict, List, Optional @@ -18,18 +20,15 @@ from pipecat.metrics.metrics import LLMTokenUsage from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.llm_service import FunctionCallFromLLM from pipecat.services.openai.llm import OpenAILLMService +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator from pipecat.utils.tracing.service_decorators import traced_llm class SambaNovaLLMService(OpenAILLMService): # type: ignore """A service for interacting with SambaNova using the OpenAI-compatible interface. + This service extends OpenAILLMService to connect to SambaNova's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - Args: - api_key (str): The API key for accessing SambaNova API. - model (str, optional): The model identifier to use. Defaults to "Meta-Llama-3.3-70B-Instruct". - base_url (str, optional): The base URL for SambaNova API. Defaults to "https://api.sambanova.ai/v1". - **kwargs: Additional keyword arguments passed to OpenAILLMService. """ def __init__( @@ -40,6 +39,14 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore base_url: str = "https://api.sambanova.ai/v1", **kwargs: Dict[Any, Any], ) -> None: + """Initialize SambaNova LLM service. + + Args: + api_key: The API key for accessing SambaNova API. + model: The model identifier to use. Defaults to "Llama-4-Maverick-17B-128E-Instruct". + base_url: The base URL for SambaNova API. Defaults to "https://api.sambanova.ai/v1". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client( @@ -48,16 +55,31 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore base_url: Optional[str] = None, **kwargs: Dict[Any, Any], ) -> Any: - """Create OpenAI-compatible client for SambaNova API endpoint.""" + """Create OpenAI-compatible client for SambaNova API endpoint. + Args: + api_key: API key for authentication. If None, uses instance default. + base_url: Base URL for the API endpoint. If None, uses instance default. + **kwargs: Additional keyword arguments for client configuration. + + Returns: + Configured OpenAI-compatible client instance. + """ logger.debug(f"Creating SambaNova client with API {base_url}") return super().create_client(api_key, base_url, **kwargs) async def get_chat_completions( self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] ) -> Any: - """Get chat completions from SambaNova API endpoint.""" + """Get chat completions from SambaNova API endpoint. + Args: + context: OpenAI LLM context containing tools and configuration. + messages: List of chat completion message parameters. + + Returns: + Chat completion response stream from SambaNova API. + """ params = { "model": self.model_name, "stream": True, @@ -78,8 +100,18 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore @traced_llm # type: ignore async def _process_context(self, context: OpenAILLMContext) -> AsyncStream[ChatCompletionChunk]: - """Redefine this method until SambaNova API introduces indexing in tool calls.""" + """Process OpenAI LLM context and stream chat completion chunks. + This method handles the streaming response from SambaNova API, including + function call processing and text frame generation. It includes special + handling for SambaNova's API limitations with tool call indexing. + + Args: + context: OpenAI LLM context containing conversation state and tools. + + Returns: + Async stream of chat completion chunks. + """ functions_list = [] arguments_list = [] tool_id_list = [] @@ -94,7 +126,7 @@ class SambaNovaLLMService(OpenAILLMService): # type: ignore context ) - async for chunk in chunk_stream: + async for chunk in WatchdogAsyncIterator(chunk_stream, manager=self.task_manager): if chunk.usage: tokens = LLMTokenUsage( prompt_tokens=chunk.usage.prompt_tokens, diff --git a/src/pipecat/services/sambanova/stt.py b/src/pipecat/services/sambanova/stt.py index ed518d6b8..71a709420 100644 --- a/src/pipecat/services/sambanova/stt.py +++ b/src/pipecat/services/sambanova/stt.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""SambaNova's Speech-to-Text service implementation for real-time transcription.""" + from typing import Any, Optional from pipecat.services.whisper.base_stt import BaseWhisperSTTService, Transcription @@ -12,16 +14,9 @@ from pipecat.transcriptions.language import Language class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore """SambaNova Whisper speech-to-text service. + Uses SambaNova's Whisper API to convert audio to text. Requires a SambaNova API key set via the api_key parameter or SAMBANOVA_API_KEY environment variable. - Args: - model: Whisper model to use. Defaults to "Whisper-Large-v3". - api_key: SambaNova API key. Defaults to None. - base_url: API base URL. Defaults to "https://api.sambanova.ai/v1". - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to `pipecat.services.whisper.base_stt.BaseWhisperSTTService`. """ def __init__( @@ -35,6 +30,17 @@ class SambaNovaSTTService(BaseWhisperSTTService): # type: ignore temperature: Optional[float] = None, **kwargs: Any, ) -> None: + """Initialize SambaNova STT service. + + Args: + model: Whisper model to use. Defaults to "Whisper-Large-v3". + api_key: SambaNova API key. Defaults to None. + base_url: API base URL. Defaults to "https://api.sambanova.ai/v1". + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Optional sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to `pipecat.services.whisper.base_stt.BaseWhisperSTTService`. + """ super().__init__( model=model, api_key=api_key, diff --git a/src/pipecat/services/sarvam/tts.py b/src/pipecat/services/sarvam/tts.py index f9ce4e70f..eee4048cb 100644 --- a/src/pipecat/services/sarvam/tts.py +++ b/src/pipecat/services/sarvam/tts.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Sarvam AI text-to-speech service implementation.""" + import base64 from typing import AsyncGenerator, Optional @@ -25,7 +27,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts def language_to_sarvam_language(language: Language) -> Optional[str]: - """Convert Pipecat Language enum to Sarvam AI language codes.""" + """Convert Pipecat Language enum to Sarvam AI language codes. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding Sarvam AI language code, or None if not supported. + """ LANGUAGE_MAP = { Language.BN: "bn-IN", # Bengali Language.EN: "en-IN", # English (India) @@ -50,15 +59,6 @@ class SarvamTTSService(TTSService): Indian languages. Provides control over voice characteristics like pitch, pace, and loudness. - Args: - api_key: Sarvam AI API subscription key. - voice_id: Speaker voice ID (e.g., "anushka", "meera"). - model: TTS model to use ("bulbul:v1" or "bulbul:v2"). - aiohttp_session: Shared aiohttp session for making requests. - base_url: Sarvam AI API base URL. - sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000). - params: Additional voice and preprocessing parameters. - Example: ```python tts = SarvamTTSService( @@ -76,6 +76,16 @@ class SarvamTTSService(TTSService): """ class InputParams(BaseModel): + """Input parameters for Sarvam TTS configuration. + + Parameters: + language: Language for synthesis. Defaults to English (India). + pitch: Voice pitch adjustment (-0.75 to 0.75). Defaults to 0.0. + pace: Speech pace multiplier (0.3 to 3.0). Defaults to 1.0. + loudness: Volume multiplier (0.1 to 3.0). Defaults to 1.0. + enable_preprocessing: Whether to enable text preprocessing. Defaults to False. + """ + language: Optional[Language] = Language.EN pitch: Optional[float] = Field(default=0.0, ge=-0.75, le=0.75) pace: Optional[float] = Field(default=1.0, ge=0.3, le=3.0) @@ -94,6 +104,18 @@ class SarvamTTSService(TTSService): params: Optional[InputParams] = None, **kwargs, ): + """Initialize the Sarvam TTS service. + + Args: + api_key: Sarvam AI API subscription key. + voice_id: Speaker voice ID (e.g., "anushka", "meera"). Defaults to "anushka". + model: TTS model to use ("bulbul:v1" or "bulbul:v2"). Defaults to "bulbul:v2". + aiohttp_session: Shared aiohttp session for making requests. + base_url: Sarvam AI API base URL. Defaults to "https://api.sarvam.ai". + sample_rate: Audio sample rate in Hz (8000, 16000, 22050, 24000). If None, uses default. + params: Additional voice and preprocessing parameters. If None, uses defaults. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) params = params or SarvamTTSService.InputParams() @@ -116,17 +138,43 @@ class SarvamTTSService(TTSService): self.set_voice(voice_id) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Sarvam service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to Sarvam AI language format. + + Args: + language: The language to convert. + + Returns: + The Sarvam AI-specific language code, or None if not supported. + """ return language_to_sarvam_language(language) async def start(self, frame: StartFrame): + """Start the Sarvam TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._settings["sample_rate"] = self.sample_rate @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using Sarvam AI's API. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") try: diff --git a/src/pipecat/services/simli/video.py b/src/pipecat/services/simli/video.py index 76381b9c6..6ba7da4b8 100644 --- a/src/pipecat/services/simli/video.py +++ b/src/pipecat/services/simli/video.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Simli video service for real-time avatar generation.""" + import asyncio import numpy as np @@ -18,6 +20,7 @@ from pipecat.frames.frames import ( TTSAudioRawFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, StartFrame +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator try: from av.audio.frame import AudioFrame @@ -30,12 +33,26 @@ except ModuleNotFoundError as e: class SimliVideoService(FrameProcessor): + """Simli video service for real-time avatar generation. + + Provides real-time avatar video generation by processing audio frames + and producing synchronized video output using the Simli API. Handles + audio resampling, video frame processing, and connection management. + """ + def __init__( self, simli_config: SimliConfig, use_turn_server: bool = False, latency_interval: int = 0, ): + """Initialize the Simli video service. + + Args: + simli_config: Configuration object for Simli client settings. + use_turn_server: Whether to use TURN server for connection. Defaults to False. + latency_interval: Latency interval setting for video processing. Defaults to 0. + """ super().__init__() self._simli_client = SimliClient(simli_config, use_turn_server, latency_interval) @@ -48,6 +65,7 @@ class SimliVideoService(FrameProcessor): self._video_task: asyncio.Task = None async def _start_connection(self): + """Start the connection to Simli service and begin processing tasks.""" if not self._initialized: await self._simli_client.Initialize() self._initialized = True @@ -60,9 +78,10 @@ class SimliVideoService(FrameProcessor): self._video_task = self.create_task(self._consume_and_process_video()) async def _consume_and_process_audio(self): + """Consume audio frames from Simli and push them downstream.""" await self._pipecat_resampler_event.wait() - async for audio_frame in self._simli_client.getAudioStreamIterator(): - self.start_watchdog() + audio_iterator = self._simli_client.getAudioStreamIterator() + async for audio_frame in WatchdogAsyncIterator(audio_iterator, manager=self.task_manager): resampled_frames = self._pipecat_resampler.resample(audio_frame) for resampled_frame in resampled_frames: audio_array = resampled_frame.to_ndarray() @@ -75,12 +94,12 @@ class SimliVideoService(FrameProcessor): num_channels=1, ), ) - self.reset_watchdog() async def _consume_and_process_video(self): + """Consume video frames from Simli and convert them to output frames.""" await self._pipecat_resampler_event.wait() - async for video_frame in self._simli_client.getVideoStreamIterator(targetFormat="rgb24"): - self.start_watchdog() + video_iterator = self._simli_client.getVideoStreamIterator(targetFormat="rgb24") + async for video_frame in WatchdogAsyncIterator(video_iterator, manager=self.task_manager): # Process the video frame convertedFrame: OutputImageRawFrame = OutputImageRawFrame( image=video_frame.to_rgb().to_image().tobytes(), @@ -89,9 +108,14 @@ class SimliVideoService(FrameProcessor): ) convertedFrame.pts = video_frame.pts await self.push_frame(convertedFrame) - self.reset_watchdog() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames and handle Simli video generation. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, StartFrame): await self.push_frame(frame, direction) @@ -128,6 +152,7 @@ class SimliVideoService(FrameProcessor): await self.push_frame(frame, direction) async def _stop(self): + """Stop the Simli client and cancel processing tasks.""" await self._simli_client.stop() if self._audio_task: await self.cancel_task(self._audio_task) diff --git a/src/pipecat/services/stt_service.py b/src/pipecat/services/stt_service.py index 5e57b3104..db777c77f 100644 --- a/src/pipecat/services/stt_service.py +++ b/src/pipecat/services/stt_service.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base classes for Speech-to-Text services with continuous and segmented processing.""" + import io import wave from abc import abstractmethod @@ -26,7 +28,12 @@ from pipecat.transcriptions.language import Language class STTService(AIService): - """STTService is a base class for speech-to-text services.""" + """Base class for speech-to-text services. + + Provides common functionality for STT services including audio passthrough, + muting, settings management, and audio processing. Subclasses must implement + the run_stt method to provide actual speech recognition. + """ def __init__( self, @@ -35,6 +42,15 @@ class STTService(AIService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the STT service. + + Args: + audio_passthrough: Whether to pass audio frames downstream after processing. + Defaults to True. + sample_rate: The sample rate for audio input. If None, will be determined + from the start frame. + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._audio_passthrough = audio_passthrough self._init_sample_rate = sample_rate @@ -44,25 +60,59 @@ class STTService(AIService): @property def is_muted(self) -> bool: - """Returns whether the STT service is currently muted.""" + """Check if the STT service is currently muted. + + Returns: + True if the service is muted and will not process audio. + """ return self._muted @property def sample_rate(self) -> int: + """Get the current sample rate for audio processing. + + Returns: + The sample rate in Hz. + """ return self._sample_rate async def set_model(self, model: str): + """Set the speech recognition model. + + Args: + model: The name of the model to use for speech recognition. + """ self.set_model_name(model) async def set_language(self, language: Language): + """Set the language for speech recognition. + + Args: + language: The language to use for speech recognition. + """ pass @abstractmethod async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Returns transcript as a string""" + """Run speech-to-text on the provided audio data. + + This method must be implemented by subclasses to provide actual speech + recognition functionality. + + Args: + audio: Raw audio bytes to transcribe. + + Yields: + Frame: Frames containing transcription results (typically TextFrame). + """ pass async def start(self, frame: StartFrame): + """Start the STT service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._sample_rate = self._init_sample_rate or frame.audio_in_sample_rate @@ -80,13 +130,24 @@ class STTService(AIService): logger.warning(f"Unknown setting for STT service: {key}") async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): + """Process an audio frame for speech recognition. + + Args: + frame: The audio frame to process. + direction: The direction of frame processing. + """ if self._muted: return await self.process_generator(self.run_stt(frame.audio)) async def process_frame(self, frame: Frame, direction: FrameDirection): - """Processes a frame of audio data, either buffering or transcribing it.""" + """Process frames, handling VAD events and audio segmentation. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, AudioRawFrame): @@ -106,17 +167,24 @@ class STTService(AIService): class SegmentedSTTService(STTService): - """SegmentedSTTService is an STTService that uses VAD events to detect - speech and will run speech-to-text on speech segments only, instead of a - continous stream. Since it uses VAD it means that VAD needs to be enabled in - the pipeline. + """STT service that processes speech in segments using VAD events. - This service always keeps a small audio buffer to take into account that VAD - events are delayed from when the user speech really starts. + Uses Voice Activity Detection (VAD) events to detect speech segments and runs + speech-to-text only on those segments, rather than continuously. + Requires VAD to be enabled in the pipeline to function properly. Maintains a + small audio buffer to account for the delay between actual speech start and + VAD detection. """ def __init__(self, *, sample_rate: Optional[int] = None, **kwargs): + """Initialize the segmented STT service. + + Args: + sample_rate: The sample rate for audio input. If None, will be determined + from the start frame. + **kwargs: Additional arguments passed to the parent STTService. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._content = None self._wave = None @@ -125,10 +193,16 @@ class SegmentedSTTService(STTService): self._user_speaking = False async def start(self, frame: StartFrame): + """Start the segmented STT service and initialize audio buffer. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._audio_buffer_size_1s = self.sample_rate * 2 async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames, handling VAD events and audio segmentation.""" await super().process_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): @@ -162,6 +236,15 @@ class SegmentedSTTService(STTService): self._audio_buffer.clear() async def process_audio_frame(self, frame: AudioRawFrame, direction: FrameDirection): + """Process audio frames by buffering them for segmented transcription. + + Continuously buffers audio, growing the buffer while user is speaking and + maintaining a small buffer when not speaking to account for VAD delay. + + Args: + frame: The audio frame to process. + direction: The direction of frame processing. + """ # If the user is speaking the audio buffer will keep growing. self._audio_buffer += frame.audio diff --git a/src/pipecat/services/tavus/video.py b/src/pipecat/services/tavus/video.py index 4ec744c51..999d712d0 100644 --- a/src/pipecat/services/tavus/video.py +++ b/src/pipecat/services/tavus/video.py @@ -4,7 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""This module implements Tavus as a sink transport layer""" +"""Tavus video service implementation for avatar-based video generation. + +This module implements Tavus as a sink transport layer, providing video +avatar functionality through Tavus's streaming API. +""" import asyncio from typing import Optional @@ -27,26 +31,19 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup from pipecat.services.ai_service import AIService from pipecat.transports.services.tavus import TavusCallbacks, TavusParams, TavusTransportClient +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue class TavusVideoService(AIService): - """ - Service class that proxies audio to Tavus and receives both audio and video in return. + """Service that proxies audio to Tavus and receives audio and video in return. - It uses the `TavusTransportClient` to manage the session and handle communication. When - audio is sent, Tavus responds with both audio and video streams, which are then routed - through Pipecat’s media pipeline. + Uses the TavusTransportClient to manage sessions and handle communication. + When audio is sent, Tavus responds with both audio and video streams, which + are routed through Pipecat's media pipeline. - In use cases such as with `DailyTransport`, this results in two distinct virtual rooms: - - **Tavus room**: Contains the Tavus Avatar and the Pipecat Bot. - - **User room**: Contains the Pipecat Bot and the user. - - Args: - api_key (str): Tavus API key used for authentication. - replica_id (str): ID of the Tavus voice replica to use for speech synthesis. - persona_id (str): ID of the Tavus persona. Defaults to "pipecat-stream" to use the Pipecat TTS voice. - session (aiohttp.ClientSession): Async HTTP session used for communication with Tavus. - **kwargs: Additional arguments passed to the parent `AIService` class. + In use cases with DailyTransport, this creates two distinct virtual rooms: + - Tavus room: Contains the Tavus Avatar and the Pipecat Bot + - User room: Contains the Pipecat Bot and the user """ def __init__( @@ -58,6 +55,15 @@ class TavusVideoService(AIService): session: aiohttp.ClientSession, **kwargs, ) -> None: + """Initialize the Tavus video service. + + Args: + api_key: Tavus API key used for authentication. + replica_id: ID of the Tavus voice replica to use for speech synthesis. + persona_id: ID of the Tavus persona. Defaults to "pipecat-stream" for Pipecat TTS voice. + session: Async HTTP session used for communication with Tavus. + **kwargs: Additional arguments passed to the parent AIService class. + """ super().__init__(**kwargs) self._api_key = api_key self._session = session @@ -71,12 +77,16 @@ class TavusVideoService(AIService): self._resampler = create_default_resampler() self._audio_buffer = bytearray() - self._queue = asyncio.Queue() self._send_task: Optional[asyncio.Task] = None # This is the custom track destination expected by Tavus self._transport_destination: Optional[str] = "stream" async def setup(self, setup: FrameProcessorSetup): + """Set up the Tavus video service. + + Args: + setup: Frame processor setup configuration. + """ await super().setup(setup) callbacks = TavusCallbacks( on_participant_joined=self._on_participant_joined, @@ -99,15 +109,18 @@ class TavusVideoService(AIService): await self._client.setup(setup) async def cleanup(self): + """Clean up the service and release resources.""" await super().cleanup() await self._client.cleanup() self._client = None async def _on_participant_left(self, participant, reason): + """Handle participant leaving the session.""" participant_id = participant["id"] logger.info(f"Participant left {participant_id}, reason: {reason}") async def _on_participant_joined(self, participant): + """Handle participant joining the session.""" participant_id = participant["id"] logger.info(f"Participant joined {participant_id}") if not self._other_participant_has_joined: @@ -124,6 +137,7 @@ class TavusVideoService(AIService): async def _on_participant_video_frame( self, participant_id: str, video_frame: VideoFrame, video_source: str ): + """Handle incoming video frames from participants.""" frame = OutputImageRawFrame( image=video_frame.buffer, size=(video_frame.width, video_frame.height), @@ -135,6 +149,7 @@ class TavusVideoService(AIService): async def _on_participant_audio_data( self, participant_id: str, audio: AudioData, audio_source: str ): + """Handle incoming audio data from participants.""" frame = OutputAudioRawFrame( audio=audio.audio_frames, sample_rate=audio.sample_rate, @@ -144,12 +159,27 @@ class TavusVideoService(AIService): await self.push_frame(frame) def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as Tavus service supports metrics generation. + """ return True async def get_persona_name(self) -> str: + """Get the name of the current persona. + + Returns: + The persona name from the Tavus client. + """ return await self._client.get_persona_name() async def start(self, frame: StartFrame): + """Start the Tavus video service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) await self._client.start(frame) if self._transport_destination: @@ -157,16 +187,32 @@ class TavusVideoService(AIService): await self._create_send_task() async def stop(self, frame: EndFrame): + """Stop the Tavus video service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._end_conversation() await self._cancel_send_task() async def cancel(self, frame: CancelFrame): + """Cancel the Tavus video service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._end_conversation() await self._cancel_send_task() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames through the service. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, StartInterruptionFrame): @@ -178,25 +224,30 @@ class TavusVideoService(AIService): await self.push_frame(frame, direction) async def _handle_interruptions(self): + """Handle interruption events by resetting send tasks and notifying client.""" await self._cancel_send_task() await self._create_send_task() await self._client.send_interrupt_message() async def _end_conversation(self): + """End the current conversation and reset state.""" await self._client.stop() self._other_participant_has_joined = False async def _create_send_task(self): + """Create the audio sending task if it doesn't exist.""" if not self._send_task: - self._queue = asyncio.Queue() + self._queue = WatchdogQueue(self.task_manager) self._send_task = self.create_task(self._send_task_handler()) async def _cancel_send_task(self): + """Cancel the audio sending task if it exists.""" if self._send_task: await self.cancel_task(self._send_task) self._send_task = None async def _handle_audio_frame(self, frame: OutputAudioRawFrame): + """Process audio frames for sending to Tavus.""" sample_rate = self._client.out_sample_rate # 40 ms of audio chunk_size = int((sample_rate * 2) / 25) @@ -215,9 +266,9 @@ class TavusVideoService(AIService): self._audio_buffer = self._audio_buffer[chunk_size:] async def _send_task_handler(self): + """Handle sending audio frames to the Tavus client.""" while True: frame = await self._queue.get() - self.start_watchdog() if isinstance(frame, OutputAudioRawFrame) and self._client: await self._client.write_audio_frame(frame) - self.reset_watchdog() + self._queue.task_done() diff --git a/src/pipecat/services/together/llm.py b/src/pipecat/services/together/llm.py index 31b15ae73..7a22c885a 100644 --- a/src/pipecat/services/together/llm.py +++ b/src/pipecat/services/together/llm.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Together.ai LLM service implementation using OpenAI-compatible interface.""" + from loguru import logger from pipecat.services.openai.llm import OpenAILLMService @@ -14,12 +16,6 @@ class TogetherLLMService(OpenAILLMService): This service extends OpenAILLMService to connect to Together.ai's API endpoint while maintaining full compatibility with OpenAI's interface and functionality. - - Args: - api_key (str): The API key for accessing Together.ai's API - base_url (str, optional): The base URL for Together.ai API. Defaults to "https://api.together.xyz/v1" - model (str, optional): The model identifier to use. Defaults to "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo" - **kwargs: Additional keyword arguments passed to OpenAILLMService """ def __init__( @@ -30,9 +26,26 @@ class TogetherLLMService(OpenAILLMService): model: str = "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo", **kwargs, ): + """Initialize Together.ai LLM service. + + Args: + api_key: The API key for accessing Together.ai's API. + base_url: The base URL for Together.ai API. Defaults to "https://api.together.xyz/v1". + model: The model identifier to use. Defaults to "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo". + **kwargs: Additional keyword arguments passed to OpenAILLMService. + """ super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) def create_client(self, api_key=None, base_url=None, **kwargs): - """Create OpenAI-compatible client for Together.ai API endpoint.""" + """Create OpenAI-compatible client for Together.ai API endpoint. + + Args: + api_key: The API key to use for the client. If None, uses instance api_key. + base_url: The base URL for the API. If None, uses instance base_url. + **kwargs: Additional keyword arguments passed to the parent create_client method. + + Returns: + An OpenAI-compatible client configured for Together.ai's API. + """ logger.debug(f"Creating Together.ai client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) diff --git a/src/pipecat/services/tts_service.py b/src/pipecat/services/tts_service.py index 904f603a9..183ef1f19 100644 --- a/src/pipecat/services/tts_service.py +++ b/src/pipecat/services/tts_service.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base classes for Text-to-speech services.""" + import asyncio from abc import abstractmethod from typing import Any, AsyncGenerator, Dict, List, Mapping, Optional, Sequence, Tuple @@ -35,6 +37,7 @@ from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService from pipecat.services.websocket_service import WebsocketService from pipecat.transcriptions.language import Language +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue from pipecat.utils.text.base_text_aggregator import BaseTextAggregator from pipecat.utils.text.base_text_filter import BaseTextFilter from pipecat.utils.text.simple_text_aggregator import SimpleTextAggregator @@ -42,6 +45,13 @@ from pipecat.utils.time import seconds_to_nanoseconds class TTSService(AIService): + """Base class for text-to-speech services. + + Provides common functionality for TTS services including text aggregation, + filtering, audio generation, and frame management. Supports configurable + sentence aggregation, silence insertion, and frame processing control. + """ + def __init__( self, *, @@ -70,6 +80,23 @@ class TTSService(AIService): transport_destination: Optional[str] = None, **kwargs, ): + """Initialize the TTS service. + + Args: + aggregate_sentences: Whether to aggregate text into sentences before synthesis. + push_text_frames: Whether to push TextFrames and LLMFullResponseEndFrames. + push_stop_frames: Whether to automatically push TTSStoppedFrames. + stop_frame_timeout_s: Idle time before pushing TTSStoppedFrame when push_stop_frames is True. + push_silence_after_stop: Whether to push silence audio after TTSStoppedFrame. + silence_time_s: Duration of silence to push when push_silence_after_stop is True. + pause_frame_processing: Whether to pause frame processing during audio generation. + sample_rate: Output sample rate for generated audio. + text_aggregator: Custom text aggregator for processing incoming text. + text_filters: Sequence of text filters to apply after aggregation. + text_filter: Single text filter (deprecated, use text_filters). + transport_destination: Destination for generated audio frames. + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._aggregate_sentences: bool = aggregate_sentences self._push_text_frames: bool = push_text_frames @@ -104,54 +131,113 @@ class TTSService(AIService): @property def sample_rate(self) -> int: + """Get the current sample rate for audio output. + + Returns: + The sample rate in Hz. + """ return self._sample_rate @property def chunk_size(self) -> int: - """This property indicates how much audio we download (from TTS services + """Get the recommended chunk size for audio streaming. + + This property indicates how much audio we download (from TTS services that require chunking) before we start pushing the first audio frame. This will make sure we download the rest of the audio while audio is being played without causing audio glitches (specially at the beginning). Of course, this will also depend on how fast the TTS service generates bytes. + Returns: + The recommended chunk size in bytes. """ CHUNK_SECONDS = 0.5 return int(self.sample_rate * CHUNK_SECONDS * 2) # 2 bytes/sample async def set_model(self, model: str): + """Set the TTS model to use. + + Args: + model: The name of the TTS model. + """ self.set_model_name(model) def set_voice(self, voice: str): + """Set the voice for speech synthesis. + + Args: + voice: The voice identifier or name. + """ self._voice_id = voice # Converts the text to audio. @abstractmethod async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Run text-to-speech synthesis on the provided text. + + This method must be implemented by subclasses to provide actual TTS functionality. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ pass def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a language to the service-specific language format. + + Args: + language: The language to convert. + + Returns: + The service-specific language identifier, or None if not supported. + """ return Language(language) async def update_setting(self, key: str, value: Any): + """Update a service-specific setting. + + Args: + key: The setting key to update. + value: The new value for the setting. + """ pass async def flush_audio(self): + """Flush any buffered audio data.""" pass async def start(self, frame: StartFrame): + """Start the TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._sample_rate = self._init_sample_rate or frame.audio_out_sample_rate if self._push_stop_frames and not self._stop_frame_task: self._stop_frame_task = self.create_task(self._stop_frame_handler()) async def stop(self, frame: EndFrame): + """Stop the TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) if self._stop_frame_task: await self.cancel_task(self._stop_frame_task) self._stop_frame_task = None async def cancel(self, frame: CancelFrame): + """Cancel the TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) if self._stop_frame_task: await self.cancel_task(self._stop_frame_task) @@ -175,9 +261,23 @@ class TTSService(AIService): logger.warning(f"Unknown setting for TTS service: {key}") async def say(self, text: str): + """Immediately speak the provided text. + + Args: + text: The text to speak. + """ await self.queue_frame(TTSSpeakFrame(text)) async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames for text-to-speech conversion. + + Handles TextFrames for synthesis, interruption frames, settings updates, + and various control frames. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if ( @@ -222,6 +322,12 @@ class TTSService(AIService): await self.push_frame(frame, direction) async def push_frame(self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM): + """Push a frame downstream with TTS-specific handling. + + Args: + frame: The frame to push. + direction: The direction to push the frame. + """ if self._push_silence_after_stop and isinstance(frame, TTSStoppedFrame): silence_num_bytes = int(self._silence_time_s * self.sample_rate * 2) # 16-bit silence_frame = TTSAudioRawFrame( @@ -315,46 +421,80 @@ class TTSService(AIService): if has_started: await self.push_frame(TTSStoppedFrame()) has_started = False + finally: + self.reset_watchdog() class WordTTSService(TTSService): - """This is a base class for TTS services that support word timestamps. Word - timestamps are useful to synchronize audio with text of the spoken - words. This way only the spoken words are added to the conversation context. + """Base class for TTS services that support word timestamps. + Word timestamps are useful to synchronize audio with text of the spoken + words. This way only the spoken words are added to the conversation context. """ def __init__(self, **kwargs): + """Initialize the Word TTS service. + + Args: + **kwargs: Additional arguments passed to the parent TTSService. + """ super().__init__(**kwargs) self._initial_word_timestamp = -1 - self._words_queue = asyncio.Queue() self._words_task = None self._llm_response_started: bool = False def start_word_timestamps(self): + """Start tracking word timestamps from the current time.""" if self._initial_word_timestamp == -1: self._initial_word_timestamp = self.get_clock().get_time() def reset_word_timestamps(self): + """Reset word timestamp tracking.""" self._initial_word_timestamp = -1 async def add_word_timestamps(self, word_times: List[Tuple[str, float]]): + """Add word timestamps to the processing queue. + + Args: + word_times: List of (word, timestamp) tuples where timestamp is in seconds. + """ for word, timestamp in word_times: await self._words_queue.put((word, seconds_to_nanoseconds(timestamp))) async def start(self, frame: StartFrame): + """Start the word TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._create_words_task() async def stop(self, frame: EndFrame): + """Stop the word TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) await self._stop_words_task() async def cancel(self, frame: CancelFrame): + """Cancel the word TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._stop_words_task() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with word timestamp awareness. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, LLMFullResponseStartFrame): @@ -369,6 +509,7 @@ class WordTTSService(TTSService): def _create_words_task(self): if not self._words_task: + self._words_queue = WatchdogQueue(self.task_manager) self._words_task = self.create_task(self._words_task_handler()) async def _stop_words_task(self): @@ -400,18 +541,29 @@ class WordTTSService(TTSService): class WebsocketTTSService(TTSService, WebsocketService): - """This is a base class for websocket-based TTS services. + """Base class for websocket-based TTS services. - If an error occurs with the websocket, an "on_connection_error" event will - be triggered: + Combines TTS functionality with websocket connectivity, providing automatic + error handling and reconnection capabilities. - @tts.event_handler("on_connection_error") - async def on_connection_error(tts: TTSService, error: str): - ... + Event handlers: + on_connection_error: Called when a websocket connection error occurs. + Example: + ```python + @tts.event_handler("on_connection_error") + async def on_connection_error(tts: TTSService, error: str): + logger.error(f"TTS connection error: {error}") + ``` """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): + """Initialize the Websocket TTS service. + + Args: + reconnect_on_error: Whether to automatically reconnect on websocket errors. + **kwargs: Additional arguments passed to parent classes. + """ TTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) self._register_event_handler("on_connection_error") @@ -422,13 +574,18 @@ class WebsocketTTSService(TTSService, WebsocketService): class InterruptibleTTSService(WebsocketTTSService): - """This is a base class for websocket-based TTS services that don't support - word timestamps and that don't offer a way to correlate the generated audio - to the requested text. + """Websocket-based TTS service that handles interruptions without word timestamps. + Designed for TTS services that don't support word timestamps. Handles interruptions + by reconnecting the websocket when the bot is speaking and gets interrupted. """ def __init__(self, **kwargs): + """Initialize the Interruptible TTS service. + + Args: + **kwargs: Additional arguments passed to the parent WebsocketTTSService. + """ super().__init__(**kwargs) # Indicates if the bot is speaking. If the bot is not speaking we don't @@ -443,6 +600,12 @@ class InterruptibleTTSService(WebsocketTTSService): await self._connect() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with bot speaking state tracking. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, BotStartedSpeakingFrame): @@ -452,19 +615,28 @@ class InterruptibleTTSService(WebsocketTTSService): class WebsocketWordTTSService(WordTTSService, WebsocketService): - """This is a base class for websocket-based TTS services that support word - timestamps. + """Base class for websocket-based TTS services that support word timestamps. - If an error occurs with the websocket a "on_connection_error" event will be - triggered: + Combines word timestamp functionality with websocket connectivity. - @tts.event_handler("on_connection_error") - async def on_connection_error(tts: TTSService, error: str): - ... + Event handlers: + on_connection_error: Called when a websocket connection error occurs. + Example: + ```python + @tts.event_handler("on_connection_error") + async def on_connection_error(tts: TTSService, error: str): + logger.error(f"TTS connection error: {error}") + ``` """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): + """Initialize the Websocket Word TTS service. + + Args: + reconnect_on_error: Whether to automatically reconnect on websocket errors. + **kwargs: Additional arguments passed to parent classes. + """ WordTTSService.__init__(self, **kwargs) WebsocketService.__init__(self, reconnect_on_error=reconnect_on_error, **kwargs) self._register_event_handler("on_connection_error") @@ -475,13 +647,18 @@ class WebsocketWordTTSService(WordTTSService, WebsocketService): class InterruptibleWordTTSService(WebsocketWordTTSService): - """This is a base class for websocket-based TTS services that support word - timestamps but don't offer a way to correlate the generated audio to the - requested text. + """Websocket-based TTS service with word timestamps that handles interruptions. + For TTS services that support word timestamps but can't correlate generated + audio with requested text. Handles interruptions by reconnecting when needed. """ def __init__(self, **kwargs): + """Initialize the Interruptible Word TTS service. + + Args: + **kwargs: Additional arguments passed to the parent WebsocketWordTTSService. + """ super().__init__(**kwargs) # Indicates if the bot is speaking. If the bot is not speaking we don't @@ -496,6 +673,12 @@ class InterruptibleWordTTSService(WebsocketWordTTSService): await self._connect() async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames with bot speaking state tracking. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, BotStartedSpeakingFrame): @@ -505,7 +688,9 @@ class InterruptibleWordTTSService(WebsocketWordTTSService): class AudioContextWordTTSService(WebsocketWordTTSService): - """This is a base class for websocket-based TTS services that support word + """Websocket-based TTS service with word timestamps and audio context management. + + This is a base class for websocket-based TTS services that support word timestamps and also allow correlating the generated audio with the requested text. @@ -516,23 +701,35 @@ class AudioContextWordTTSService(WebsocketWordTTSService): The audio received from the TTS will be played in context order. That is, if we requested audio for a context "A" and then audio for context "B", the audio from context ID "A" will be played first. - """ def __init__(self, **kwargs): + """Initialize the Audio Context Word TTS service. + + Args: + **kwargs: Additional arguments passed to the parent WebsocketWordTTSService. + """ super().__init__(**kwargs) - self._contexts_queue = asyncio.Queue() self._contexts: Dict[str, asyncio.Queue] = {} self._audio_context_task = None async def create_audio_context(self, context_id: str): - """Create a new audio context.""" + """Create a new audio context for grouping related audio. + + Args: + context_id: Unique identifier for the audio context. + """ await self._contexts_queue.put(context_id) self._contexts[context_id] = asyncio.Queue() logger.trace(f"{self} created audio context {context_id}") async def append_to_audio_context(self, context_id: str, frame: TTSAudioRawFrame): - """Append audio to an existing context.""" + """Append audio to an existing context. + + Args: + context_id: The context to append audio to. + frame: The audio frame to append. + """ if self.audio_context_available(context_id): logger.trace(f"{self} appending audio {frame} to audio context {context_id}") await self._contexts[context_id].put(frame) @@ -540,7 +737,11 @@ class AudioContextWordTTSService(WebsocketWordTTSService): logger.warning(f"{self} unable to append audio to context {context_id}") async def remove_audio_context(self, context_id: str): - """Remove an existing audio context.""" + """Remove an existing audio context. + + Args: + context_id: The context to remove. + """ if self.audio_context_available(context_id): # We just mark the audio context for deletion by appending # None. Once we reach None while handling audio we know we can @@ -551,14 +752,31 @@ class AudioContextWordTTSService(WebsocketWordTTSService): logger.warning(f"{self} unable to remove context {context_id}") def audio_context_available(self, context_id: str) -> bool: - """Checks whether the given audio context is registered.""" + """Check whether the given audio context is registered. + + Args: + context_id: The context ID to check. + + Returns: + True if the context exists and is available. + """ return context_id in self._contexts async def start(self, frame: StartFrame): + """Start the audio context TTS service. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) self._create_audio_context_task() async def stop(self, frame: EndFrame): + """Stop the audio context TTS service. + + Args: + frame: The end frame. + """ await super().stop(frame) if self._audio_context_task: # Indicate no more audio contexts are available. this will end the @@ -568,6 +786,11 @@ class AudioContextWordTTSService(WebsocketWordTTSService): self._audio_context_task = None async def cancel(self, frame: CancelFrame): + """Cancel the audio context TTS service. + + Args: + frame: The cancel frame. + """ await super().cancel(frame) await self._stop_audio_context_task() @@ -578,7 +801,7 @@ class AudioContextWordTTSService(WebsocketWordTTSService): def _create_audio_context_task(self): if not self._audio_context_task: - self._contexts_queue = asyncio.Queue() + self._contexts_queue = WatchdogQueue(self.task_manager) self._contexts: Dict[str, asyncio.Queue] = {} self._audio_context_task = self.create_task(self._audio_context_task_handler()) @@ -620,10 +843,12 @@ class AudioContextWordTTSService(WebsocketWordTTSService): while running: try: frame = await asyncio.wait_for(queue.get(), timeout=AUDIO_CONTEXT_TIMEOUT) + self.reset_watchdog() if frame: await self.push_frame(frame) running = frame is not None except asyncio.TimeoutError: + self.reset_watchdog() # We didn't get audio, so let's consider this context finished. logger.trace(f"{self} time out on audio context {context_id}") break diff --git a/src/pipecat/services/ultravox/stt.py b/src/pipecat/services/ultravox/stt.py index 8021f3c25..987593f02 100644 --- a/src/pipecat/services/ultravox/stt.py +++ b/src/pipecat/services/ultravox/stt.py @@ -44,13 +44,12 @@ except ModuleNotFoundError as e: class AudioBuffer: """Buffer to collect audio frames before processing. - Attributes: - frames: List of AudioRawFrames to process - started_at: Timestamp when speech started - is_processing: Flag to prevent concurrent processing + Manages the collection and state of audio frames during speech + recording sessions, including timing and processing flags. """ def __init__(self): + """Initialize the audio buffer.""" self.frames: List[AudioRawFrame] = [] self.started_at: Optional[float] = None self.is_processing: bool = False @@ -59,19 +58,17 @@ class AudioBuffer: class UltravoxModel: """Model wrapper for the Ultravox multimodal model. - This class handles loading and running the Ultravox model for speech-to-text. - - Args: - model_name: The name or path of the Ultravox model to load - - Attributes: - model_name: The name of the loaded model - engine: The vLLM engine for model inference - tokenizer: The tokenizer for the model - stop_token_ids: Optional token IDs to stop generation + This class handles loading and running the Ultravox model for speech-to-text + transcription using vLLM for efficient inference. """ def __init__(self, model_name: str = "fixie-ai/ultravox-v0_5-llama-3_1-8b"): + """Initialize the Ultravox model. + + Args: + model_name: The name or path of the Ultravox model to load. + Defaults to "fixie-ai/ultravox-v0_5-llama-3_1-8b". + """ self.model_name = model_name self._initialize_engine() self._initialize_tokenizer() @@ -95,10 +92,10 @@ class UltravoxModel: """Format chat messages into a prompt for the model. Args: - messages: List of message dictionaries with 'role' and 'content' + messages: List of message dictionaries with 'role' and 'content'. Returns: - str: Formatted prompt string + str: Formatted prompt string ready for model input. """ return self.tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True @@ -114,13 +111,13 @@ class UltravoxModel: """Generate text from audio input using the model. Args: - messages: List of message dictionaries - temperature: Sampling temperature - max_tokens: Maximum tokens to generate - audio: Audio data as numpy array + messages: List of message dictionaries for conversation context. + temperature: Sampling temperature for generation randomness. + max_tokens: Maximum number of tokens to generate. + audio: Audio data as numpy array in float32 format. Yields: - str: JSON chunks of the generated response + str: JSON chunks of the generated response in OpenAI format. """ sampling_params = SamplingParams( temperature=temperature, max_tokens=max_tokens, stop_token_ids=self.stop_token_ids @@ -173,22 +170,9 @@ class UltravoxModel: class UltravoxSTTService(AIService): """Service to transcribe audio using the Ultravox multimodal model. - This service collects audio frames and processes them with Ultravox - to generate text transcriptions. - - Args: - model_name: The Ultravox model to use (ModelSize enum or string) - hf_token: Hugging Face token for model access - temperature: Sampling temperature for generation - max_tokens: Maximum tokens to generate - **kwargs: Additional arguments passed to AIService - - Attributes: - model: The UltravoxModel instance - buffer: Buffer to collect audio frames - temperature: Temperature for text generation - max_tokens: Maximum tokens to generate - _connection_active: Flag indicating if service is active + This service collects audio frames during speech and processes them with + Ultravox to generate text transcriptions. It handles real-time audio + buffering, model warm-up, and streaming text generation. """ def __init__( @@ -200,6 +184,17 @@ class UltravoxSTTService(AIService): max_tokens: int = 100, **kwargs, ): + """Initialize the UltravoxSTTService. + + Args: + model_name: The Ultravox model to use. Defaults to + "fixie-ai/ultravox-v0_5-llama-3_1-8b". + hf_token: Hugging Face token for model access. If None, will try + to use HF_TOKEN environment variable. + temperature: Sampling temperature for generation. Defaults to 0.7. + max_tokens: Maximum tokens to generate. Defaults to 100. + **kwargs: Additional arguments passed to AIService. + """ super().__init__(**kwargs) # Authenticate with Hugging Face if token provided @@ -283,8 +278,11 @@ class UltravoxSTTService(AIService): async def start(self, frame: StartFrame): """Handle service start. + Starts the service, marks it as active, and performs model warm-up + to ensure optimal performance for the first inference. + Args: - frame: StartFrame that triggered this method + frame: StartFrame that triggered this method. """ await super().start(frame) self._connection_active = True @@ -296,8 +294,10 @@ class UltravoxSTTService(AIService): async def stop(self, frame: EndFrame): """Handle service stop. + Stops the service and marks it as inactive. + Args: - frame: EndFrame that triggered this method + frame: EndFrame that triggered this method. """ await super().stop(frame) self._connection_active = False @@ -306,8 +306,10 @@ class UltravoxSTTService(AIService): async def cancel(self, frame: CancelFrame): """Handle service cancellation. + Cancels the service, clears any buffered audio, and marks it as inactive. + Args: - frame: CancelFrame that triggered this method + frame: CancelFrame that triggered this method. """ await super().cancel(frame) self._connection_active = False @@ -317,11 +319,12 @@ class UltravoxSTTService(AIService): async def process_frame(self, frame: Frame, direction: FrameDirection): """Process incoming frames. - This method collects audio frames and processes them when speech ends. + This method collects audio frames during speech and processes them + when speech ends to generate text transcriptions. Args: - frame: The frame to process - direction: Direction of the frame (input/output) + frame: The frame to process. + direction: Direction of the frame (input/output). """ await super().process_frame(frame, direction) diff --git a/src/pipecat/services/vision_service.py b/src/pipecat/services/vision_service.py index 23eb79c4e..e245e9a01 100644 --- a/src/pipecat/services/vision_service.py +++ b/src/pipecat/services/vision_service.py @@ -4,6 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Vision service implementation. + +Provides base classes and implementations for computer vision services that can +analyze images and generate textual descriptions or answers to questions about +visual content. +""" + from abc import abstractmethod from typing import AsyncGenerator @@ -13,17 +20,49 @@ from pipecat.services.ai_service import AIService class VisionService(AIService): - """VisionService is a base class for vision services.""" + """Base class for vision services. + + Provides common functionality for vision services that process images and + generate textual responses. Handles image frame processing and integrates + with the AI service infrastructure for metrics and lifecycle management. + """ def __init__(self, **kwargs): + """Initialize the vision service. + + Args: + **kwargs: Additional arguments passed to the parent AIService. + """ super().__init__(**kwargs) self._describe_text = None @abstractmethod async def run_vision(self, frame: VisionImageRawFrame) -> AsyncGenerator[Frame, None]: + """Process a vision image frame and generate results. + + This method must be implemented by subclasses to provide actual computer + vision functionality such as image description, object detection, or + visual question answering. + + Args: + frame: The vision image frame to process, containing image data. + + Yields: + Frame: Frames containing the vision analysis results, typically TextFrame + objects with descriptions or answers. + """ pass async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames, handling vision image frames for analysis. + + Automatically processes VisionImageRawFrame objects by calling run_vision + and handles metrics tracking. Other frames are passed through unchanged. + + Args: + frame: The frame to process. + direction: The direction of frame processing. + """ await super().process_frame(frame, direction) if isinstance(frame, VisionImageRawFrame): diff --git a/src/pipecat/services/websocket_service.py b/src/pipecat/services/websocket_service.py index d757946f9..a58a654d9 100644 --- a/src/pipecat/services/websocket_service.py +++ b/src/pipecat/services/websocket_service.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base websocket service with automatic reconnection and error handling.""" + import asyncio from abc import ABC, abstractmethod from typing import Awaitable, Callable, Optional @@ -17,18 +19,28 @@ from pipecat.utils.network import exponential_backoff_time class WebsocketService(ABC): - """Base class for websocket-based services with reconnection logic.""" + """Base class for websocket-based services with automatic reconnection. + + Provides websocket connection management, automatic reconnection with + exponential backoff, connection verification, and error handling. + Subclasses implement service-specific connection and message handling logic. + """ def __init__(self, *, reconnect_on_error: bool = True, **kwargs): - """Initialize websocket attributes.""" + """Initialize the websocket service. + + Args: + reconnect_on_error: Whether to automatically reconnect on connection errors. + **kwargs: Additional arguments (unused, for compatibility). + """ self._websocket: Optional[websockets.WebSocketClientProtocol] = None self._reconnect_on_error = reconnect_on_error async def _verify_connection(self) -> bool: - """Verify websocket connection is working. + """Verify the websocket connection is active and responsive. Returns: - bool: True if connection is verified working, False otherwise + True if connection is verified working, False otherwise. """ try: if not self._websocket or self._websocket.closed: @@ -40,13 +52,13 @@ class WebsocketService(ABC): return False async def _reconnect_websocket(self, attempt_number: int) -> bool: - """Reconnect the websocket. + """Reconnect the websocket with the current attempt number. Args: - attempt_number: Current retry attempt number + attempt_number: Current retry attempt number for logging. Returns: - bool: True if reconnection and verification successful, False otherwise + True if reconnection and verification successful, False otherwise. """ logger.warning(f"{self} reconnecting (attempt: {attempt_number})") await self._disconnect_websocket() @@ -54,10 +66,14 @@ class WebsocketService(ABC): return await self._verify_connection() async def _receive_task_handler(self, report_error: Callable[[ErrorFrame], Awaitable[None]]): - """Handles WebSocket message receiving with automatic retry logic. + """Handle websocket message receiving with automatic retry logic. + + Continuously receives messages with automatic reconnection on errors. + Uses exponential backoff between retry attempts and reports fatal errors + after maximum retries are exhausted. Args: - report_error: Callback to report errors + report_error: Callback function to report connection errors. """ retry_count = 0 MAX_RETRIES = 3 @@ -98,33 +114,45 @@ class WebsocketService(ABC): @abstractmethod async def _connect(self): - """Implement service-specific connection logic. This function will - connect to the websocket via _connect_websocket() among other connection - logic.""" + """Connect to the service. + + Implement service-specific connection logic including websocket connection + via _connect_websocket() and any additional setup required. + """ pass @abstractmethod async def _disconnect(self): - """Implement service-specific disconnection logic. This function will - disconnect to the websocket via _connect_websocket() among other - connection logic. + """Disconnect from the service. + Implement service-specific disconnection logic including websocket + disconnection via _disconnect_websocket() and any cleanup required. """ pass @abstractmethod async def _connect_websocket(self): - """Implement service-specific websocket connection logic. This function - should only connect to the websocket.""" + """Establish the websocket connection. + + Implement the low-level websocket connection logic specific to the service. + Should only handle websocket connection, not additional service setup. + """ pass @abstractmethod async def _disconnect_websocket(self): - """Implement service-specific websocket disconnection logic. This - function should only disconnect from the websocket.""" + """Close the websocket connection. + + Implement the low-level websocket disconnection logic specific to the service. + Should only handle websocket disconnection, not additional service cleanup. + """ pass @abstractmethod async def _receive_messages(self): - """Implement service-specific message receiving logic.""" + """Receive and process websocket messages. + + Implement service-specific logic for receiving and handling messages + from the websocket connection. Called continuously by the receive task handler. + """ pass diff --git a/src/pipecat/services/whisper/base_stt.py b/src/pipecat/services/whisper/base_stt.py index 018dae85a..6f8ac26f2 100644 --- a/src/pipecat/services/whisper/base_stt.py +++ b/src/pipecat/services/whisper/base_stt.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Base class for Whisper-based speech-to-text services. + +This module provides common functionality for services implementing the Whisper API +interface, including language mapping, metrics generation, and error handling. +""" + from typing import AsyncGenerator, Optional from loguru import logger @@ -18,9 +24,16 @@ from pipecat.utils.tracing.service_decorators import traced_stt def language_to_whisper_language(language: Language) -> Optional[str]: - """Language support for Whisper API. + """Maps pipecat Language enum to Whisper API language codes. + Language support for Whisper API. Docs: https://platform.openai.com/docs/guides/speech-to-text#supported-languages + + Args: + language: A Language enum value representing the input language. + + Returns: + str or None: The corresponding Whisper language code, or None if not supported. """ BASE_LANGUAGES = { Language.AF: "af", @@ -98,15 +111,6 @@ class BaseWhisperSTTService(SegmentedSTTService): Provides common functionality for services implementing the Whisper API interface, including metrics generation and error handling. - - Args: - model: Name of the Whisper model to use. - api_key: Service API key. Defaults to None. - base_url: Service API base URL. Defaults to None. - language: Language of the audio input. Defaults to English. - prompt: Optional text to guide the model's style or continue a previous segment. - temperature: Sampling temperature between 0 and 1. Defaults to 0.0. - **kwargs: Additional arguments passed to SegmentedSTTService. """ def __init__( @@ -120,6 +124,17 @@ class BaseWhisperSTTService(SegmentedSTTService): temperature: Optional[float] = None, **kwargs, ): + """Initialize the Whisper STT service. + + Args: + model: Name of the Whisper model to use. + api_key: Service API key. Defaults to None. + base_url: Service API base URL. Defaults to None. + language: Language of the audio input. Defaults to English. + prompt: Optional text to guide the model's style or continue a previous segment. + temperature: Sampling temperature between 0 and 1. Defaults to 0.0. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ super().__init__(**kwargs) self.set_model_name(model) self._client = self._create_client(api_key, base_url) @@ -138,12 +153,30 @@ class BaseWhisperSTTService(SegmentedSTTService): return AsyncOpenAI(api_key=api_key, base_url=base_url) async def set_model(self, model: str): + """Set the model name for transcription. + + Args: + model: The name of the model to use. + """ self.set_model_name(model) def can_generate_metrics(self) -> bool: + """Indicates whether this service can generate metrics. + + Returns: + bool: True, as this service supports metric generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert from pipecat Language to service language code. + + Args: + language: The Language enum value to convert. + + Returns: + str or None: The corresponding service language code, or None if not supported. + """ return language_to_whisper_language(language) async def set_language(self, language: Language): @@ -163,6 +196,15 @@ class BaseWhisperSTTService(SegmentedSTTService): pass async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: + """Transcribe audio data to text. + + Args: + audio: Raw audio data to transcribe. + + Yields: + Frame: Either a TranscriptionFrame containing the transcribed text + or an ErrorFrame if transcription fails. + """ try: await self.start_processing_metrics() await self.start_ttfb_metrics() diff --git a/src/pipecat/services/whisper/stt.py b/src/pipecat/services/whisper/stt.py index d6920ed6c..ace18ab56 100644 --- a/src/pipecat/services/whisper/stt.py +++ b/src/pipecat/services/whisper/stt.py @@ -4,7 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -"""This module implements Whisper transcription with a locally-downloaded model.""" +"""Whisper speech-to-text services with locally-downloaded models. + +This module implements Whisper transcription using locally-downloaded models, +supporting both Faster Whisper and MLX Whisper backends for efficient inference. +""" import asyncio from enum import Enum @@ -37,18 +41,18 @@ if TYPE_CHECKING: class Model(Enum): - """Class of basic Whisper model selection options. + """Whisper model selection options for Faster Whisper. - Available models: - Multilingual models: - TINY: Smallest multilingual model - BASE: Basic multilingual model - MEDIUM: Good balance for multilingual - LARGE: Best quality multilingual - DISTIL_LARGE_V2: Fast multilingual + Provides various model sizes and specializations for speech recognition, + balancing quality and performance based on use case requirements. - English-only models: - DISTIL_MEDIUM_EN: Fast English-only + Parameters: + TINY: Smallest multilingual model, fastest inference. + BASE: Basic multilingual model, good speed/quality balance. + MEDIUM: Medium-sized multilingual model, better quality. + LARGE: Best quality multilingual model, slower inference. + DISTIL_LARGE_V2: Fast multilingual distilled model. + DISTIL_MEDIUM_EN: Fast English-only distilled model. """ # Multilingual models @@ -63,16 +67,18 @@ class Model(Enum): class MLXModel(Enum): - """Class of MLX Whisper model selection options. + """MLX Whisper model selection options for Apple Silicon. - Available models: - Multilingual models: - TINY: Smallest multilingual model - MEDIUM: Good balance for multilingual - LARGE_V3: Best quality multilingual - LARGE_V3_TURBO: Finetuned, pruned Whisper large-v3, much faster, slightly lower quality - DISTIL_LARGE_V3: Fast multilingual - LARGE_V3_TURBO_Q4: LARGE_V3_TURBO, quantized to Q4 + Provides various model sizes optimized for Apple Silicon hardware, + including quantized variants for improved performance. + + Parameters: + TINY: Smallest multilingual model for MLX. + MEDIUM: Medium-sized multilingual model for MLX. + LARGE_V3: Best quality multilingual model for MLX. + LARGE_V3_TURBO: Finetuned, pruned Whisper large-v3, much faster with slightly lower quality. + DISTIL_LARGE_V3: Fast multilingual distilled model for MLX. + LARGE_V3_TURBO_Q4: LARGE_V3_TURBO quantized to Q4 for reduced memory usage. """ # Multilingual models @@ -256,21 +262,6 @@ class WhisperSTTService(SegmentedSTTService): This service uses Faster Whisper to perform speech-to-text transcription on audio segments. It supports multiple languages and various model sizes. - - Args: - model: The Whisper model to use for transcription. Can be a Model enum or string. - device: The device to run inference on ('cpu', 'cuda', or 'auto'). - compute_type: The compute type for inference ('default', 'int8', 'int8_float16', etc.). - no_speech_prob: Probability threshold for filtering out non-speech segments. - language: The default language for transcription. - **kwargs: Additional arguments passed to SegmentedSTTService. - - Attributes: - _device: The device used for inference. - _compute_type: The compute type for inference. - _no_speech_prob: Threshold for non-speech filtering. - _model: The loaded Whisper model instance. - _settings: Dictionary containing service settings. """ def __init__( @@ -283,6 +274,16 @@ class WhisperSTTService(SegmentedSTTService): language: Language = Language.EN, **kwargs, ): + """Initialize the Whisper STT service. + + Args: + model: The Whisper model to use for transcription. Can be a Model enum or string. + device: The device to run inference on ('cpu', 'cuda', or 'auto'). + compute_type: The compute type for inference ('default', 'int8', 'int8_float16', etc.). + no_speech_prob: Probability threshold for filtering out non-speech segments. + language: The default language for transcription. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ super().__init__(**kwargs) self._device: str = device self._compute_type = compute_type @@ -355,7 +356,7 @@ class WhisperSTTService(SegmentedSTTService): pass async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Transcribes given audio using Whisper. + """Transcribe audio data using Whisper. Args: audio: Raw audio bytes in 16-bit PCM format. @@ -402,18 +403,6 @@ class WhisperSTTServiceMLX(WhisperSTTService): This service uses MLX Whisper to perform speech-to-text transcription on audio segments. It's optimized for Apple Silicon and supports multiple languages and quantizations. - - Args: - model: The MLX Whisper model to use for transcription. Can be an MLXModel enum or string. - no_speech_prob: Probability threshold for filtering out non-speech segments. - language: The default language for transcription. - temperature: Temperature for sampling. Can be a float or tuple of floats. - **kwargs: Additional arguments passed to SegmentedSTTService. - - Attributes: - _no_speech_threshold: Threshold for non-speech filtering. - _temperature: Temperature for sampling. - _settings: Dictionary containing service settings. """ def __init__( @@ -425,6 +414,15 @@ class WhisperSTTServiceMLX(WhisperSTTService): temperature: float = 0.0, **kwargs, ): + """Initialize the MLX Whisper STT service. + + Args: + model: The MLX Whisper model to use for transcription. Can be an MLXModel enum or string. + no_speech_prob: Probability threshold for filtering out non-speech segments. + language: The default language for transcription. + temperature: Temperature for sampling. Can be a float or tuple of floats. + **kwargs: Additional arguments passed to SegmentedSTTService. + """ # Skip WhisperSTTService.__init__ and call its parent directly SegmentedSTTService.__init__(self, **kwargs) @@ -455,7 +453,10 @@ class WhisperSTTServiceMLX(WhisperSTTService): @override async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """Transcribes given audio using MLX Whisper. + """Transcribe audio data using MLX Whisper. + + The audio is expected to be 16-bit signed PCM data. + MLX Whisper will handle the conversion internally. Args: audio: Raw audio bytes in 16-bit PCM format. @@ -463,10 +464,6 @@ class WhisperSTTServiceMLX(WhisperSTTService): Yields: Frame: Either a TranscriptionFrame containing the transcribed text or an ErrorFrame if transcription fails. - - Note: - The audio is expected to be 16-bit signed PCM data. - MLX Whisper will handle the conversion internally. """ try: import mlx_whisper diff --git a/src/pipecat/services/xtts/tts.py b/src/pipecat/services/xtts/tts.py index 2111e4d72..6332e26ef 100644 --- a/src/pipecat/services/xtts/tts.py +++ b/src/pipecat/services/xtts/tts.py @@ -4,6 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""XTTS text-to-speech service implementation. + +This module provides integration with Coqui XTTS streaming server for +text-to-speech synthesis using local Docker deployment. +""" + from typing import Any, AsyncGenerator, Dict, Optional import aiohttp @@ -31,6 +37,14 @@ from pipecat.utils.tracing.service_decorators import traced_tts def language_to_xtts_language(language: Language) -> Optional[str]: + """Convert a Language enum to XTTS language code. + + Args: + language: The Language enum value to convert. + + Returns: + The corresponding XTTS language code, or None if not supported. + """ BASE_LANGUAGES = { Language.CS: "cs", Language.DE: "de", @@ -70,6 +84,13 @@ def language_to_xtts_language(language: Language) -> Optional[str]: class XTTSService(TTSService): + """Coqui XTTS text-to-speech service. + + Provides text-to-speech synthesis using a locally running Coqui XTTS + streaming server. Supports multiple languages and voice cloning through + studio speakers configuration. + """ + def __init__( self, *, @@ -80,6 +101,16 @@ class XTTSService(TTSService): sample_rate: Optional[int] = None, **kwargs, ): + """Initialize the XTTS service. + + Args: + voice_id: ID of the voice/speaker to use for synthesis. + base_url: Base URL of the XTTS streaming server. + aiohttp_session: HTTP session for making requests to the server. + language: Language for synthesis. Defaults to English. + sample_rate: Audio sample rate. If None, uses default. + **kwargs: Additional arguments passed to parent TTSService. + """ super().__init__(sample_rate=sample_rate, **kwargs) self._settings = { @@ -93,12 +124,30 @@ class XTTSService(TTSService): self._resampler = create_default_resampler() def can_generate_metrics(self) -> bool: + """Check if this service can generate processing metrics. + + Returns: + True, as XTTS service supports metrics generation. + """ return True def language_to_service_language(self, language: Language) -> Optional[str]: + """Convert a Language enum to XTTS service language format. + + Args: + language: The language to convert. + + Returns: + The XTTS-specific language code, or None if not supported. + """ return language_to_xtts_language(language) async def start(self, frame: StartFrame): + """Start the XTTS service and load studio speakers. + + Args: + frame: The start frame containing initialization parameters. + """ await super().start(frame) if self._studio_speakers: @@ -120,6 +169,14 @@ class XTTSService(TTSService): @traced_tts async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + """Generate speech from text using XTTS streaming server. + + Args: + text: The text to synthesize into speech. + + Yields: + Frame: Audio frames containing the synthesized speech. + """ logger.debug(f"{self}: Generating TTS [{text}]") if not self._studio_speakers: diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index a6de4f46e..a2f269309 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -4,13 +4,23 @@ # SPDX-License-Identifier: BSD 2-Clause License # +"""Language code enumerations for Pipecat. + +This module provides comprehensive language code constants following ISO 639 +and BCP 47 standards, supporting both language-only and language-region +combinations for various speech and text processing services. +""" + import sys from enum import Enum if sys.version_info < (3, 11): class StrEnum(str, Enum): + """String enumeration base class for Python < 3.11 compatibility.""" + def __new__(cls, value): + """Create a new instance of the StrEnum.""" obj = str.__new__(cls, value) obj._value_ = value return obj @@ -19,6 +29,14 @@ else: class Language(StrEnum): + """Language codes for speech and text processing services. + + Provides comprehensive language code constants following ISO 639 and BCP 47 + standards. Includes both language-only codes (e.g., 'en') and language-region + combinations (e.g., 'en-US') to support various speech synthesis, recognition, + and translation services. + """ + # Afrikaans AF = "af" AF_ZA = "af-ZA" diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index d707c5969..93cd90825 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -368,8 +368,6 @@ class BaseInputTransport(FrameProcessor): self._audio_in_queue.get(), timeout=AUDIO_INPUT_TIMEOUT_SECS ) - self.start_watchdog() - # If an audio filter is available, run it before VAD. if self._params.audio_in_filter: frame.audio = await self._params.audio_in_filter.filter(frame.audio) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 386f223d7..36d0536d7 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -39,6 +39,7 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.base_transport import TransportParams +from pipecat.utils.asyncio.watchdog_priority_queue import WatchdogPriorityQueue from pipecat.utils.time import nanoseconds_to_seconds BOT_VAD_STOP_SECS = 0.35 @@ -441,8 +442,10 @@ class BaseOutputTransport(FrameProcessor): frame = await asyncio.wait_for( self._audio_queue.get(), timeout=vad_stop_secs ) + self._transport.reset_watchdog() yield frame except asyncio.TimeoutError: + self._transport.reset_watchdog() # Notify the bot stopped speaking upstream if necessary. await self._bot_stopped_speaking() @@ -452,11 +455,13 @@ class BaseOutputTransport(FrameProcessor): while True: try: frame = self._audio_queue.get_nowait() + self._transport.reset_watchdog() if isinstance(frame, OutputAudioRawFrame): frame.audio = await self._mixer.mix(frame.audio) last_frame_time = time.time() yield frame except asyncio.QueueEmpty: + self._transport.reset_watchdog() # Notify the bot stopped speaking upstream if necessary. diff_time = time.time() - last_frame_time if diff_time > vad_stop_secs: @@ -597,7 +602,7 @@ class BaseOutputTransport(FrameProcessor): def _create_clock_task(self): if not self._clock_task: - self._clock_queue = asyncio.PriorityQueue() + self._clock_queue = WatchdogPriorityQueue(self._transport.task_manager) self._clock_task = self._transport.create_task(self._clock_task_handler()) async def _cancel_clock_task(self): diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index 7f6256b83..5ddaacff7 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -26,11 +26,12 @@ from pipecat.frames.frames import ( TransportMessageFrame, TransportMessageUrgentFrame, ) -from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSetup +from pipecat.processors.frame_processor import FrameDirection from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator try: from fastapi import WebSocket @@ -178,12 +179,12 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): async def _receive_messages(self): try: - async for message in self._client.receive(): + async for message in WatchdogAsyncIterator( + self._client.receive(), manager=self.task_manager + ): if not self._params.serializer: continue - self.start_watchdog() - frame = await self._params.serializer.deserialize(message) if not frame: @@ -193,13 +194,9 @@ class FastAPIWebsocketInputTransport(BaseInputTransport): await self.push_audio_frame(frame) else: await self.push_frame(frame) - - self.reset_watchdog() except Exception as e: logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})") - self.reset_watchdog() - await self._client.trigger_client_disconnected() async def _monitor_websocket(self): diff --git a/src/pipecat/transports/network/small_webrtc.py b/src/pipecat/transports/network/small_webrtc.py index 5b36fec37..9eedd7d95 100644 --- a/src/pipecat/transports/network/small_webrtc.py +++ b/src/pipecat/transports/network/small_webrtc.py @@ -33,6 +33,7 @@ from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator try: import cv2 @@ -422,19 +423,22 @@ class SmallWebRTCInputTransport(BaseInputTransport): async def _receive_audio(self): try: - async for audio_frame in self._client.read_audio_frame(): - self.start_watchdog() + audio_iterator = self._client.read_audio_frame() + async for audio_frame in WatchdogAsyncIterator( + audio_iterator, manager=self.task_manager + ): if audio_frame: await self.push_audio_frame(audio_frame) - self.reset_watchdog() except Exception as e: logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})") async def _receive_video(self): try: - async for video_frame in self._client.read_video_frame(): - self.start_watchdog() + video_iterator = self._client.read_video_frame() + async for video_frame in WatchdogAsyncIterator( + video_iterator, manager=self.task_manager + ): if video_frame: await self.push_video_frame(video_frame) @@ -453,7 +457,6 @@ class SmallWebRTCInputTransport(BaseInputTransport): await self.push_video_frame(image_frame) # Remove from pending requests del self._image_requests[req_id] - self.reset_watchdog() except Exception as e: logger.error(f"{self} exception receiving data: {e.__class__.__name__} ({e})") diff --git a/src/pipecat/transports/network/websocket_client.py b/src/pipecat/transports/network/websocket_client.py index 738904546..98c7f9e2d 100644 --- a/src/pipecat/transports/network/websocket_client.py +++ b/src/pipecat/transports/network/websocket_client.py @@ -30,7 +30,7 @@ from pipecat.serializers.protobuf import ProtobufFrameSerializer from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.asyncio import BaseTaskManager +from pipecat.utils.asyncio.task_manager import BaseTaskManager class WebsocketClientParams(TransportParams): diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 776da1693..4c00fa44c 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -39,7 +39,8 @@ from pipecat.transcriptions.language import Language from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.asyncio import BaseTaskManager +from pipecat.utils.asyncio.task_manager import BaseTaskManager +from pipecat.utils.asyncio.watchdog_queue import WatchdogQueue try: from daily import ( @@ -304,6 +305,7 @@ class DailyTransportClient(EventHandler): self._leave_counter = 0 self._task_manager: Optional[BaseTaskManager] = None + self._watchdog_timers_enabled = False # We use the executor to cleanup the client. We just do it from one # place, so only one thread is really needed. @@ -320,9 +322,6 @@ class DailyTransportClient(EventHandler): # waits for it to finish using completions (and a future) we will # deadlock because completions use event handlers (which are holding the # GIL). - self._event_queue = asyncio.Queue() - self._audio_queue = asyncio.Queue() - self._video_queue = asyncio.Queue() self._event_task = None self._audio_task = None self._video_task = None @@ -400,6 +399,9 @@ class DailyTransportClient(EventHandler): return self._task_manager = setup.task_manager + self._watchdog_timers_enabled = setup.watchdog_timers_enabled + + self._event_queue = WatchdogQueue(self._task_manager) self._event_task = self._task_manager.create_task( self._callback_task_handler(self._event_queue), f"{self}::event_callback_task", @@ -424,12 +426,14 @@ class DailyTransportClient(EventHandler): self._out_sample_rate = self._params.audio_out_sample_rate or frame.audio_out_sample_rate if self._params.audio_in_enabled and not self._audio_task and self._task_manager: + self._audio_queue = WatchdogQueue(self._task_manager) self._audio_task = self._task_manager.create_task( self._callback_task_handler(self._audio_queue), f"{self}::audio_callback_task", ) if self._params.video_in_enabled and not self._video_task and self._task_manager: + self._video_queue = WatchdogQueue(self._task_manager) self._video_task = self._task_manager.create_task( self._callback_task_handler(self._video_queue), f"{self}::video_callback_task", @@ -934,6 +938,7 @@ class DailyTransportClient(EventHandler): await self._joined_event.wait() (callback, *args) = await queue.get() await callback(*args) + queue.task_done() def _get_event_loop(self) -> asyncio.AbstractEventLoop: if not self._task_manager: diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index ecb2718c8..53dd091ef 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -27,7 +27,8 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessorSet from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from pipecat.utils.asyncio import BaseTaskManager +from pipecat.utils.asyncio.task_manager import BaseTaskManager +from pipecat.utils.asyncio.watchdog_async_iterator import WatchdogAsyncIterator try: from livekit import rtc @@ -341,8 +342,9 @@ class LiveKitTransportClient: logger.warning(f"Received unexpected event type: {type(event)}") async def get_next_audio_frame(self): - frame, participant_id = await self._audio_queue.get() - return frame, participant_id + while True: + frame, participant_id = await self._audio_queue.get() + yield frame, participant_id def __str__(self): return f"{self._transport_name}::LiveKitTransportClient" @@ -413,9 +415,8 @@ class LiveKitInputTransport(BaseInputTransport): async def _audio_in_task_handler(self): logger.info("Audio input task started") - while True: - audio_data = await self._client.get_next_audio_frame() - self.start_watchdog() + audio_iterator = self._client.get_next_audio_frame() + async for audio_data in WatchdogAsyncIterator(audio_iterator, manager=self.task_manager): if audio_data: audio_frame_event, participant_id = audio_data pipecat_audio_frame = await self._convert_livekit_audio_to_pipecat( @@ -428,7 +429,6 @@ class LiveKitInputTransport(BaseInputTransport): num_channels=pipecat_audio_frame.num_channels, ) await self.push_audio_frame(input_audio_frame) - self.reset_watchdog() async def _convert_livekit_audio_to_pipecat( self, audio_frame_event: rtc.AudioFrameEvent diff --git a/src/pipecat/utils/asyncio/__init__.py b/src/pipecat/utils/asyncio/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/utils/asyncio.py b/src/pipecat/utils/asyncio/task_manager.py similarity index 73% rename from src/pipecat/utils/asyncio.py rename to src/pipecat/utils/asyncio/task_manager.py index 36479edd4..844536186 100644 --- a/src/pipecat/utils/asyncio.py +++ b/src/pipecat/utils/asyncio/task_manager.py @@ -8,7 +8,7 @@ import asyncio import time from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import Coroutine, Dict, List, Optional, Sequence +from typing import Coroutine, Dict, Optional, Sequence from loguru import logger @@ -18,6 +18,7 @@ WATCHDOG_TIMEOUT = 5.0 @dataclass class TaskManagerParams: loop: asyncio.AbstractEventLoop + enable_watchdog_timers: bool = False enable_watchdog_logging: bool = False watchdog_timeout: float = WATCHDOG_TIMEOUT @@ -27,10 +28,6 @@ class BaseTaskManager(ABC): def setup(self, params: TaskManagerParams): pass - @abstractmethod - async def cleanup(self): - pass - @abstractmethod def get_event_loop(self) -> asyncio.AbstractEventLoop: pass @@ -42,6 +39,7 @@ class BaseTaskManager(ABC): name: str, *, enable_watchdog_logging: Optional[bool] = None, + enable_watchdog_timers: Optional[bool] = None, watchdog_timeout: Optional[float] = None, ) -> asyncio.Task: """ @@ -54,6 +52,7 @@ class BaseTaskManager(ABC): coroutine (Coroutine): The coroutine to be executed within the task. name (str): The name to assign to the task for identification. enable_watchdog_logging(bool): whether this task should log watchdog processing times. + enable_watchdog_timers(bool): whether this task should have a watchdog timer. watchdog_timeout(float): watchdog timer timeout for this task. Returns: @@ -98,50 +97,39 @@ class BaseTaskManager(ABC): pass @abstractmethod - def start_watchdog(self, task: asyncio.Task): - """Starts the given task watchdog timer. If not reset, a warning will be - logged indicating the task is stalling. + def task_reset_watchdog(self): + """Resets the running task watchdog timer. If not reset, a warning will + be logged indicating the task is stalling. """ pass + @property @abstractmethod - def reset_watchdog(self, task: asyncio.Task): - """Resets the given task watchdog timer. If not reset, a warning will be - logged indicating the task is stalling. - - """ + def task_watchdog_enabled(self) -> bool: + """Whether the current running task has a watchdog timer enabled.""" pass @dataclass class TaskData: task: asyncio.Task - watchdog_start: asyncio.Event watchdog_timer: asyncio.Event enable_watchdog_logging: bool + enable_watchdog_timers: bool watchdog_timeout: float + watchdog_task: Optional[asyncio.Task] class TaskManager(BaseTaskManager): def __init__(self) -> None: self._tasks: Dict[str, TaskData] = {} self._params: Optional[TaskManagerParams] = None - self._watchdog_tasks: List[asyncio.Task] = [] def setup(self, params: TaskManagerParams): if not self._params: self._params = params - async def cleanup(self): - for task in self._watchdog_tasks: - try: - task.cancel() - await task - except asyncio.CancelledError: - # This is expected, no need to re-raise. - pass - def get_event_loop(self) -> asyncio.AbstractEventLoop: if not self._params: raise Exception("TaskManager is not setup: unable to get event loop") @@ -153,6 +141,7 @@ class TaskManager(BaseTaskManager): name: str, *, enable_watchdog_logging: Optional[bool] = None, + enable_watchdog_timers: Optional[bool] = None, watchdog_timeout: Optional[float] = None, ) -> asyncio.Task: """ @@ -165,6 +154,7 @@ class TaskManager(BaseTaskManager): coroutine (Coroutine): The coroutine to be executed within the task. name (str): The name to assign to the task for identification. enable_watchdog_logging(bool): whether this task should log watchdog processing time. + enable_watchdog_timers(bool): whether this task should have a watchdog timer. watchdog_timeout(float): watchdog timer timeout for this task. Returns: @@ -186,20 +176,26 @@ class TaskManager(BaseTaskManager): task = self._params.loop.create_task(run_coroutine()) task.set_name(name) + task.add_done_callback(self._task_done_handler) self._add_task( TaskData( task=task, - watchdog_start=asyncio.Event(), watchdog_timer=asyncio.Event(), enable_watchdog_logging=( enable_watchdog_logging if enable_watchdog_logging else self._params.enable_watchdog_logging ), + enable_watchdog_timers=( + enable_watchdog_timers + if enable_watchdog_timers + else self._params.enable_watchdog_timers + ), watchdog_timeout=( watchdog_timeout if watchdog_timeout else self._params.watchdog_timeout ), - ) + watchdog_task=None, + ), ) logger.trace(f"{name}: task created") return task @@ -230,8 +226,6 @@ class TaskManager(BaseTaskManager): raise except Exception as e: logger.exception(f"{name}: unexpected exception while stopping task: {e}") - finally: - self._remove_task(task) async def cancel_task(self, task: asyncio.Task, timeout: Optional[float] = None): """Cancels the given asyncio Task and awaits its completion with an @@ -264,82 +258,69 @@ class TaskManager(BaseTaskManager): except BaseException as e: logger.critical(f"{name}: fatal base exception while cancelling task: {e}") raise - finally: - self._remove_task(task) + + def reset_watchdog(self, task: asyncio.Task): + name = task.get_name() + if name in self._tasks and self._tasks[name].enable_watchdog_timers: + self._tasks[name].watchdog_timer.set() def current_tasks(self) -> Sequence[asyncio.Task]: """Returns the list of currently created/registered tasks.""" return [data.task for data in self._tasks.values()] - def start_watchdog(self, task: asyncio.Task): - """Starts the given task watchdog timer. If not reset, a warning will be - logged indicating the task is stalling. If the timer was already started - a warning will be logged. + def task_reset_watchdog(self): + """Resets the running task watchdog timer. If not reset on time, a warning + will be logged indicating the task is stalling. """ - name = task.get_name() - if name in self._tasks: - if self._tasks[name].watchdog_start.is_set(): - logger.warning(f"Watchdog timer for task {name} already started") - else: - self._tasks[name].watchdog_timer.clear() - self._tasks[name].watchdog_start.set() - else: - logger.warning(f"Unable to start watchdog timer: task {name} does not exist") + task = asyncio.current_task() + if task: + self.reset_watchdog(task) - def reset_watchdog(self, task: asyncio.Task): - """Resets the given task watchdog timer. If not reset, a warning will be - logged indicating the task is stalling. - - """ + @property + def task_watchdog_enabled(self) -> bool: + task = asyncio.current_task() + if not task: + return False name = task.get_name() - if name in self._tasks: - self._tasks[name].watchdog_start.clear() - self._tasks[name].watchdog_timer.set() - else: - logger.warning(f"Unable to reset watchdog timer: task {name} does not exist") + return name in self._tasks and self._tasks[name].enable_watchdog_timers def _add_task(self, task_data: TaskData): name = task_data.task.get_name() self._tasks[name] = task_data - watchdog_task = self.get_event_loop().create_task( - self._watchdog_task_handler(self._tasks[name]) - ) - self._watchdog_tasks.append(watchdog_task) - - def _remove_task(self, task: asyncio.Task): - name = task.get_name() - try: - del self._tasks[name] - except KeyError as e: - logger.trace(f"{name}: unable to remove task (already removed?): {e}") + if self._params and task_data.enable_watchdog_timers: + watchdog_task = self.get_event_loop().create_task( + self._watchdog_task_handler(task_data) + ) + task_data.watchdog_task = watchdog_task async def _watchdog_task_handler(self, task_data: TaskData): name = task_data.task.get_name() - start = task_data.watchdog_start timer = task_data.watchdog_timer enable_watchdog_logging = task_data.enable_watchdog_logging watchdog_timeout = task_data.watchdog_timeout - async def wait_for_reset(): - waiting = True - while waiting: - try: - start_time = time.time() - await asyncio.wait_for(timer.wait(), timeout=watchdog_timeout) - total_time = time.time() - start_time - if enable_watchdog_logging: - logger.debug(f"{name} task processing time: {total_time:.20f}") - waiting = False - except asyncio.TimeoutError: - logger.warning( - f"{name}: task is taking too long {WATCHDOG_TIMEOUT} second(s) (forgot to reset watchdog?)" - ) - finally: - timer.clear() - while True: - # Wait for the user to start the watchdog timer. - await start.wait() - # Now, waiting for the task to finish. - await wait_for_reset() + try: + start_time = time.time() + await asyncio.wait_for(timer.wait(), timeout=watchdog_timeout) + total_time = time.time() - start_time + if enable_watchdog_logging: + logger.debug(f"{name} time between watchdog timer resets: {total_time:.20f}") + except asyncio.TimeoutError: + logger.warning( + f"{name}: task is taking too long {WATCHDOG_TIMEOUT} second(s) (forgot to reset watchdog?)" + ) + finally: + timer.clear() + + def _task_done_handler(self, task: asyncio.Task): + name = task.get_name() + try: + task_data = self._tasks[name] + if task_data.watchdog_task: + task_data.watchdog_task.cancel() + task_data.watchdog_task = None + del self._tasks[name] + except KeyError as e: + logger.trace(f"{name}: unable to remove task data (already removed?): {e}") diff --git a/src/pipecat/utils/asyncio/watchdog_async_iterator.py b/src/pipecat/utils/asyncio/watchdog_async_iterator.py new file mode 100644 index 000000000..d9d3e2f79 --- /dev/null +++ b/src/pipecat/utils/asyncio/watchdog_async_iterator.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +from typing import AsyncIterator, Optional + +from pipecat.utils.asyncio.task_manager import BaseTaskManager + + +class WatchdogAsyncIterator: + """An asynchronous iterator that monitors activity and resets the current + task watchdog timer. This is necessary to avoid task watchdog timers to + expire while we are waiting to get an item from the iterator. + + """ + + def __init__( + self, + async_iterable, + *, + manager: BaseTaskManager, + timeout: float = 2.0, + ): + self._async_iterable = async_iterable + self._manager = manager + self._timeout = timeout + self._iter: Optional[AsyncIterator] = None + self._current_anext_task: Optional[asyncio.Task] = None + + def __aiter__(self): + return self + + async def __anext__(self): + if not self._iter: + self._iter = await self._ensure_async_iterator(self._async_iterable) + + if self._manager.task_watchdog_enabled: + return await self._watchdog_anext() + else: + return await self._iter.__anext__() + + async def _watchdog_anext(self): + while True: + try: + if not self._current_anext_task: + self._current_anext_task = asyncio.create_task(self._iter.__anext__()) + + item = await asyncio.wait_for( + asyncio.shield(self._current_anext_task), + timeout=self._timeout, + ) + + self._manager.task_reset_watchdog() + + # The task has finished, so we will create a new one for the + # next item. + self._current_anext_task = None + + return item + except asyncio.TimeoutError: + self._manager.task_reset_watchdog() + except StopAsyncIteration: + self._current_anext_task = None + raise + + async def _ensure_async_iterator(self, obj) -> AsyncIterator: + aiter = obj.__aiter__() + if asyncio.iscoroutine(aiter): + aiter = await aiter + return aiter diff --git a/src/pipecat/utils/asyncio/watchdog_coroutine.py b/src/pipecat/utils/asyncio/watchdog_coroutine.py new file mode 100644 index 000000000..84855c3e6 --- /dev/null +++ b/src/pipecat/utils/asyncio/watchdog_coroutine.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +from typing import Optional + +from pipecat.utils.asyncio.task_manager import BaseTaskManager + + +class WatchdogCoroutine: + """An asynchronous iterator that monitors activity and resets the current + task watchdog timer. This is necessary to avoid task watchdog timers to + expire while we are waiting to get an item from the iterator. + + """ + + def __init__( + self, + coroutine, + *, + manager: BaseTaskManager, + timeout: float = 2.0, + ): + self._coroutine = coroutine + self._manager = manager + self._timeout = timeout + self._current_coro_task: Optional[asyncio.Task] = None + + async def __call__(self): + if self._manager.task_watchdog_enabled: + return await self._watchdog_call() + else: + return await self._coroutine + + async def _watchdog_call(self): + while True: + try: + if not self._current_coro_task: + self._current_coro_task = asyncio.create_task(self._coroutine) + + result = await asyncio.wait_for( + asyncio.shield(self._current_coro_task), + timeout=self._timeout, + ) + + self._manager.task_reset_watchdog() + + # The task has finished. + self._current_coro_task = None + + return result + except asyncio.TimeoutError: + self._manager.task_reset_watchdog() + + +async def watchdog_coroutine(coroutine, *, manager: BaseTaskManager, timeout: float = 2.0): + watchdog_coro = WatchdogCoroutine(coroutine, manager=manager, timeout=timeout) + return await watchdog_coro() diff --git a/src/pipecat/utils/asyncio/watchdog_event.py b/src/pipecat/utils/asyncio/watchdog_event.py new file mode 100644 index 000000000..65453f6ec --- /dev/null +++ b/src/pipecat/utils/asyncio/watchdog_event.py @@ -0,0 +1,42 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio + +from pipecat.utils.asyncio.task_manager import BaseTaskManager + + +class WatchdogEvent(asyncio.Event): + """An asynchronous event that resets the current task watchdog timer. This + is necessary to avoid task watchdog timers to expire while we are waiting on + the event. + + """ + + def __init__( + self, + manager: BaseTaskManager, + *, + timeout: float = 2.0, + ) -> None: + super().__init__() + self._manager = manager + self._timeout = timeout + + async def wait(self): + if self._manager.task_watchdog_enabled: + return await self._watchdog_wait() + else: + return await super().wait() + + async def _watchdog_wait(self): + while True: + try: + await asyncio.wait_for(super().wait(), timeout=self._timeout) + self._manager.task_reset_watchdog() + return True + except asyncio.TimeoutError: + self._manager.task_reset_watchdog() diff --git a/src/pipecat/utils/asyncio/watchdog_priority_queue.py b/src/pipecat/utils/asyncio/watchdog_priority_queue.py new file mode 100644 index 000000000..31d358fc7 --- /dev/null +++ b/src/pipecat/utils/asyncio/watchdog_priority_queue.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio + +from pipecat.utils.asyncio.task_manager import BaseTaskManager + + +class WatchdogPriorityQueue(asyncio.PriorityQueue): + """An asynchronous priority queue that resets the current task watchdog + timer. This is necessary to avoid task watchdog timers to expire while we + are waiting to get an item from the queue. + + """ + + def __init__( + self, + manager: BaseTaskManager, + *, + maxsize: int = 0, + timeout: float = 2.0, + ) -> None: + super().__init__(maxsize) + self._manager = manager + self._timeout = timeout + + async def get(self): + if self._manager.task_watchdog_enabled: + return await self._watchdog_get() + else: + return await super().get() + + def task_done(self): + if self._manager.task_watchdog_enabled: + self._manager.task_reset_watchdog() + super().task_done() + + async def _watchdog_get(self): + while True: + try: + item = await asyncio.wait_for(super().get(), timeout=self._timeout) + self._manager.task_reset_watchdog() + return item + except asyncio.TimeoutError: + self._manager.task_reset_watchdog() diff --git a/src/pipecat/utils/asyncio/watchdog_queue.py b/src/pipecat/utils/asyncio/watchdog_queue.py new file mode 100644 index 000000000..961324b7b --- /dev/null +++ b/src/pipecat/utils/asyncio/watchdog_queue.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio + +from pipecat.utils.asyncio.task_manager import BaseTaskManager + + +class WatchdogQueue(asyncio.Queue): + """An asynchronous queue that resets the current task watchdog timer. This + is necessary to avoid task watchdog timers to expire while we are waiting to + get an item from the queue. + + """ + + def __init__( + self, + manager: BaseTaskManager, + *, + maxsize: int = 0, + timeout: float = 2.0, + ) -> None: + super().__init__(maxsize) + self._manager = manager + self._timeout = timeout + + async def get(self): + if self._manager.task_watchdog_enabled: + return await self._watchdog_get() + else: + return await super().get() + + def task_done(self): + if self._manager.task_watchdog_enabled: + self._manager.task_reset_watchdog() + super().task_done() + + async def _watchdog_get(self): + while True: + try: + item = await asyncio.wait_for(super().get(), timeout=self._timeout) + self._manager.task_reset_watchdog() + return item + except asyncio.TimeoutError: + self._manager.task_reset_watchdog()