diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml index 5866f881b..444e24338 100644 --- a/.github/workflows/format.yaml +++ b/.github/workflows/format.yaml @@ -35,7 +35,12 @@ jobs: python -m pip install --upgrade pip pip install -r dev-requirements.txt - name: Ruff formatter - id: ruff + id: ruff-format run: | source .venv/bin/activate ruff format --diff + - name: Ruff import linter + id: ruff-check + run: | + source .venv/bin/activate + ruff check --select I diff --git a/.github/workflows/generate_docs.yaml b/.github/workflows/generate_docs.yaml deleted file mode 100644 index 242984d8a..000000000 --- a/.github/workflows/generate_docs.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Generate API Documentation - -on: - release: - types: [published] # Run on new release - workflow_dispatch: # Manual trigger - -jobs: - update-docs: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r docs/api/requirements.txt - pip install . - - - name: Generate API documentation - run: | - cd docs/api - python generate_docs.py - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 - with: - commit-message: 'docs: Update API documentation' - title: 'docs: Update API documentation' - body: | - Automated PR to update API documentation. - - - Generated using `docs/api/generate_docs.py` - - Triggered by: ${{ github.event_name }} - branch: update-api-docs - delete-branch: true - labels: | - documentation diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 667e789d9..c9176cdda 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,12 +4,33 @@ build: os: ubuntu-22.04 tools: python: '3.12' + apt_packages: + - portaudio19-dev + - python3-dev + - libasound2-dev + jobs: + pre_build: + - python -m pip install --upgrade pip + - pip install wheel setuptools + post_build: + - echo "Build completed" sphinx: configuration: docs/api/conf.py + fail_on_warning: false python: install: - requirements: docs/api/requirements.txt - method: pip path: . + +search: + ranking: + api/*: 5 + getting-started/*: 4 + guides/*: 3 + +submodules: + include: all + recursive: true diff --git a/CHANGELOG.md b/CHANGELOG.md index ac15dffec..5610525b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,18 +5,109 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.52] - 2024-12-24 ### Added +- Constructor arguments for GoogleLLMService to directly set tools and tool_config. + +- Smart turn detection example (`22d-natural-conversation-gemini-audio.py`) that + leverages Gemini 2.0 capabilities (). + (see https://x.com/kwindla/status/1870974144831275410) + +- Added `DailyTransport.send_dtmf()` to send dial-out DTMF tones. + +- Added `DailyTransport.sip_call_transfer()` to forward SIP and PSTN calls to + another address or number. For example, transfer a SIP call to a different + SIP address or transfer a PSTN phone number to a different PSTN phone number. + +- Added `DailyTransport.sip_refer()` to transfer incoming SIP/PSTN calls from + outside Daily to another SIP/PSTN address. + +- Added an `auto_mode` input parameter to `ElevenLabsTTSService`. `auto_mode` + is set to `True` by default. Enabling this setting disables the chunk + schedule and all buffers, which reduces latency. + +- Added `KoalaFilter` which implement on device noise reduction using Koala + Noise Suppression. + (see https://picovoice.ai/platform/koala/) + +- Added `CerebrasLLMService` for Cerebras integration with an OpenAI-compatible + interface. Added foundational example `14k-function-calling-cerebras.py`. + +- Pipecat now supports Python 3.13. We had a dependency on the `audioop` package + which was deprecated and now removed on Python 3.13. We are now using + `audioop-lts` (https://github.com/AbstractUmbra/audioop) to provide the same + functionality. + +- Added timestamped conversation transcript support: + + - New `TranscriptProcessor` factory provides access to user and assistant + transcript processors. + - `UserTranscriptProcessor` processes user speech with timestamps from + transcription. + - `AssistantTranscriptProcessor` processes assistant responses with LLM + context timestamps. + - Messages emitted with ISO 8601 timestamps indicating when they were spoken. + - Supports all LLM formats (OpenAI, Anthropic, Google) via standard message + format. + - New examples: `28a-transcription-processor-openai.py`, + `28b-transcription-processor-anthropic.py`, and + `28c-transcription-processor-gemini.py`. + - Add support for more languages to ElevenLabs (Arabic, Croatian, Filipino, Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa). +### Changed + +- `PlayHTTTSService` uses the new v4 websocket API, which also fixes an issue + where text inputted to the TTS didn't return audio. + +- The default model for `ElevenLabsTTSService` is now `eleven_flash_v2_5`. + +- `OpenAIRealtimeBetaLLMService` now takes a `model` parameter in the + constructor. + +- Updated the default model for the `OpenAIRealtimeBetaLLMService`. + +- Room expiration (`exp`) in `DailyRoomProperties` is now optional (`None`) by + default instead of automatically setting a 5-minute expiration time. You must + explicitly set expiration time if desired. + ### Deprecated - `AWSTTSService` is now deprecated, use `PollyTTSService` instead. +### Fixed + +- Fixed token counting in `GoogleLLMService`. Tokens were summed incorrectly + (double-counted in many cases). + +- Fixed an issue that could cause the bot to stop talking if there was a user + interruption before getting any audio from the TTS service. + +- Fixed an issue that would cause `ParallelPipeline` to handle `EndFrame` + incorrectly causing the main pipeline to not terminate or terminate too early. + +- Fixed an audio stuttering issue in `FastPitchTTSService`. + +- Fixed a `BaseOutputTransport` issue that was causing non-audio frames being + processed before the previous audio frames were played. This will allow, for + example, sending a frame `A` after a `TTSSpeakFrame` and the frame `A` will + only be pushed downstream after the audio generated from `TTSSpeakFrame` has + been spoken. + +- Fixed a `DeepgramSTTService` issue that was causing language to be passed as + an object instead of a string resulting in the connection to fail. + +## [0.0.51] - 2024-12-16 + +### Fixed + +- Fixed an issue in websocket-based TTS services that was causing infinite + reconnections (Cartesia, ElevenLabs, PlayHT and LMNT). + ## [0.0.50] - 2024-12-11 ### Added diff --git a/LICENSE b/LICENSE index cd6220df2..a457e0dfa 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 2-Clause License -Copyright (c) 2024, Daily +Copyright (c) 2025, Daily Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index b68d62778..6786c36f6 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@  pipecat -[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) +[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) Pipecat is an open source Python framework for building voice and multimodal conversational agents. It handles the complex orchestration of AI services, network transport, audio processing, and multimodal interactions, letting you focus on creating engaging experiences. @@ -55,19 +55,19 @@ pip install "pipecat-ai[option,...]" Available options include: -| Category | Services | Install Command Example | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/api-reference/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/api-reference/services/stt/azure), [Deepgram](https://docs.pipecat.ai/api-reference/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/api-reference/services/stt/gladia), [Whisper](https://docs.pipecat.ai/api-reference/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | -| LLMs | [Anthropic](https://docs.pipecat.ai/api-reference/services/llm/anthropic), [Azure](https://docs.pipecat.ai/api-reference/services/llm/azure), [Fireworks AI](https://docs.pipecat.ai/api-reference/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/api-reference/services/llm/gemini), [Grok](https://docs.pipecat.ai/api-reference/services/llm/grok), [Groq](https://docs.pipecat.ai/api-reference/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/api-reference/services/llm/nim), [Ollama](https://docs.pipecat.ai/api-reference/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/api-reference/services/llm/openai), [Together AI](https://docs.pipecat.ai/api-reference/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/api-reference/services/tts/aws), [Azure](https://docs.pipecat.ai/api-reference/services/tts/azure), [Cartesia](https://docs.pipecat.ai/api-reference/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/api-reference/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/api-reference/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/api-reference/services/tts/google), [LMNT](https://docs.pipecat.ai/api-reference/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/api-reference/services/tts/openai), [PlayHT](https://docs.pipecat.ai/api-reference/services/tts/playht), [Rime](https://docs.pipecat.ai/api-reference/services/tts/rime), [XTTS](https://docs.pipecat.ai/api-reference/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | -| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/api-reference/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/api-reference/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | -| Video | [Tavus](https://docs.pipecat.ai/api-reference/services/video/tavus), [Simli](https://docs.pipecat.ai/api-reference/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | -| Vision & Image | [Moondream](https://docs.pipecat.ai/api-reference/services/vision/moondream), [fal](https://docs.pipecat.ai/api-reference/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/api-reference/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/api-reference/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/api-reference/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | -| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/api-reference/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/api-reference/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | +| Category | Services | Install Command Example | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | +| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | +| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | +| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | -📚 [View full services documentation →](https://docs.pipecat.ai/api-reference/services/supported-services) +📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) ## Code examples @@ -197,9 +197,7 @@ You can use [use-package](https://github.com/jwiegley/use-package) to install [e :hook ((python-mode . lazy-ruff-mode)) :config (setq lazy-ruff-format-command "ruff format") - (setq lazy-ruff-only-format-block t) - (setq lazy-ruff-only-format-region t) - (setq lazy-ruff-only-format-buffer t)) + (setq lazy-ruff-check-command "ruff check --select I")) ``` `ruff` was installed in the `venv` environment described before, so you should be able to use [pyvenv-auto](https://github.com/ryotaro612/pyvenv-auto) to automatically load that environment inside Emacs. @@ -209,7 +207,6 @@ You can use [use-package](https://github.com/jwiegley/use-package) to install [e :ensure t :defer t :hook ((python-mode . pyvenv-auto-run))) - ``` ### Visual Studio Code @@ -224,6 +221,16 @@ Install the } ``` +### PyCharm + +`ruff` was installed in the `venv` environment described before, now to enable autoformatting on save, go to `File` -> `Settings` -> `Tools` -> `File Watchers` and add a new watcher with the following settings: + +1. **Name**: `Ruff formatter` +2. **File type**: `Python` +3. **Working directory**: `$ContentRoot$` +4. **Arguments**: `format $FilePath$` +5. **Program**: `$PyInterpreterDirectory$/ruff` + ## Contributing We welcome contributions from the community! Whether you're fixing bugs, improving documentation, or adding new features, here's how you can help: diff --git a/dev-requirements.txt b/dev-requirements.txt index 92b6ec4d3..0ed6d9b05 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,8 +1,9 @@ -build~=1.2.1 -grpcio-tools~=1.65.4 +build~=1.2.2 +grpcio-tools~=1.68.1 pip-tools~=7.4.1 -pyright~=1.1.376 -pytest~=8.3.2 -ruff~=0.6.7 -setuptools~=72.2.0 +pyright~=1.1.390 +pytest~=8.3.4 +ruff~=0.8.3 +setuptools~=75.6.0 setuptools_scm~=8.1.0 +python-dotenv~=1.0.1 diff --git a/docs/api/README.md b/docs/api/README.md new file mode 100644 index 000000000..22b62d45e --- /dev/null +++ b/docs/api/README.md @@ -0,0 +1,109 @@ +# Pipecat Documentation + +This directory contains the source files for auto-generating Pipecat's server API reference documentation. + +## Setup + +1. Install documentation dependencies: + +```bash +pip install -r requirements.txt +``` + +2. Make the build scripts executable: + +```bash +chmod +x build-docs.sh rtd-test.py +``` + +## Building Documentation + +From this directory, you can build the documentation in several ways: + +### Local Build + +```bash +# Using the build script (automatically opens docs when done) +./build-docs.sh + +# Or directly with sphinx-build +sphinx-build -b html . _build/html -W --keep-going +``` + +### ReadTheDocs Test Build + +To test the documentation build process exactly as it would run on ReadTheDocs: + +```bash +./rtd-test.py +``` + +This script: + +- Creates a fresh virtual environment +- Installs all dependencies as specified in requirements files +- Handles conflicting dependencies (like grpcio versions for Riva and PlayHT) +- Builds the documentation in an isolated environment +- Provides detailed logging of the build process + +Use this script to verify your documentation will build correctly on ReadTheDocs before pushing changes. + +## Viewing Documentation + +The built documentation will be available at `_build/html/index.html`. To open: + +```bash +# On MacOS +open _build/html/index.html + +# On Linux +xdg-open _build/html/index.html + +# On Windows +start _build/html/index.html +``` + +## Directory Structure + +``` +. +├── api/ # Auto-generated API documentation +├── _build/ # Built documentation +├── _static/ # Static files (images, css, etc.) +├── conf.py # Sphinx configuration +├── index.rst # Main documentation entry point +├── requirements-base.txt # Base documentation dependencies +├── requirements-riva.txt # Riva-specific dependencies +├── requirements-playht.txt # PlayHT-specific dependencies +├── build-docs.sh # Local build script +└── rtd-test.py # ReadTheDocs test build script +``` + +## Notes + +- Documentation is auto-generated from Python docstrings +- Service modules are automatically detected and included +- The build process matches our ReadTheDocs configuration +- Warnings are treated as errors (-W flag) to maintain consistency +- The --keep-going flag ensures all errors are reported +- Dependencies are split into multiple requirements files to handle version conflicts + +## Troubleshooting + +If you encounter missing service modules: + +1. Verify the service is installed with its extras: `pip install pipecat-ai[service-name]` +2. Check the build logs for import errors +3. Ensure the service module is properly initialized in the package +4. Run `./rtd-test.py` to test in an isolated environment matching ReadTheDocs + +For dependency conflicts: + +1. Check the requirements files for version specifications +2. Use `rtd-test.py` to verify dependency resolution +3. Consider adding service-specific requirements files if needed + +For more information: + +- [ReadTheDocs Configuration](.readthedocs.yaml) +- [Sphinx Documentation](https://www.sphinx-doc.org/) diff --git a/docs/api/build-docs.sh b/docs/api/build-docs.sh new file mode 100755 index 000000000..46d025404 --- /dev/null +++ b/docs/api/build-docs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Clean previous build +rm -rf _build + +# Build docs matching ReadTheDocs configuration +sphinx-build -b html -d _build/doctrees . _build/html -W --keep-going + +# Open docs (MacOS) +open _build/html/index.html \ No newline at end of file diff --git a/docs/api/conf.py b/docs/api/conf.py index fffaf90b2..36b4f4df0 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -1,6 +1,11 @@ +import logging import sys from pathlib import Path +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger("sphinx-build") + # Add source directory to path docs_dir = Path(__file__).parent project_root = docs_dir.parent.parent @@ -32,13 +37,150 @@ autodoc_default_options = { "undoc-members": True, "exclude-members": "__weakref__", "no-index": True, + "show-inheritance": True, } +# Mock imports for optional dependencies +autodoc_mock_imports = [ + "riva", + "livekit", + "pyht", # Base PlayHT package + "pyht.async_client", # PlayHT specific imports + "pyht.client", + "pyht.protos", + "pyht.protos.api_pb2", + "pipecat_ai_playht", # PlayHT wrapper + "anthropic", + "assemblyai", + "boto3", + "azure", + "cartesia", + "deepgram", + "elevenlabs", + "fal", + "gladia", + "google", + "krisp", + "langchain", + "lmnt", + "noisereduce", + "openai", + "openpipe", + "simli", + "soundfile", + # Existing mocks + "pipecat_ai_krisp", + "pyaudio", + "_tkinter", + "tkinter", + "daily", + "daily_python", + "pydantic.BaseModel", + "pydantic.Field", + "pydantic._internal._model_construction", + "pydantic._internal._fields", +] + # HTML output settings html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] autodoc_typehints = "description" -html_show_sphinx = False # Remove "Built with Sphinx" +html_show_sphinx = False + + +def verify_modules(): + """Verify that required modules are available.""" + required_modules = { + "services": [ + "assemblyai", + "aws", + "cartesia", + "deepgram", + "google", + "lmnt", + "riva", + "simli", + ], + "serializers": ["livekit"], + "vad": ["silero", "vad_analyzer"], + "transports": { + "services": ["daily", "livekit"], + "local": ["audio", "tk"], + "network": ["fastapi_websocket", "websocket_server"], + }, + } + + missing = [] + for category, modules in required_modules.items(): + if isinstance(modules, dict): + # Handle nested structure + for subcategory, submodules in modules.items(): + for module in submodules: + try: + __import__(f"pipecat.{category}.{subcategory}.{module}") + logger.info( + f"Successfully imported pipecat.{category}.{subcategory}.{module}" + ) + except (ImportError, TypeError, NameError) as e: + missing.append(f"pipecat.{category}.{subcategory}.{module}") + logger.warning( + f"Optional module not available: pipecat.{category}.{subcategory}.{module} - {str(e)}" + ) + else: + # Handle flat structure + for module in modules: + try: + __import__(f"pipecat.{category}.{module}") + logger.info(f"Successfully imported pipecat.{category}.{module}") + except (ImportError, TypeError, NameError) as e: + missing.append(f"pipecat.{category}.{module}") + logger.warning( + f"Optional module not available: pipecat.{category}.{module} - {str(e)}" + ) + + if missing: + logger.warning(f"Some optional modules are not available: {missing}") + + +def clean_title(title: str) -> str: + """Automatically clean module titles.""" + # Remove everything after space (like 'module', 'processor', etc.) + title = title.split(" ")[0] + + # Get the last part of the dot-separated path + parts = title.split(".") + title = parts[-1] + + # Special cases for service names and common acronyms + special_cases = { + "ai": "AI", + "aws": "AWS", + "api": "API", + "vad": "VAD", + "assemblyai": "AssemblyAI", + "deepgram": "Deepgram", + "elevenlabs": "ElevenLabs", + "openai": "OpenAI", + "openpipe": "OpenPipe", + "playht": "PlayHT", + "xtts": "XTTS", + "lmnt": "LMNT", + } + + # Check if the entire title is a special case + if title.lower() in special_cases: + return special_cases[title.lower()] + + # Otherwise, capitalize each word + words = title.split("_") + cleaned_words = [] + for word in words: + if word.lower() in special_cases: + cleaned_words.append(special_cases[word.lower()]) + else: + cleaned_words.append(word.capitalize()) + + return " ".join(cleaned_words) def setup(app): @@ -55,24 +197,56 @@ def setup(app): import shutil shutil.rmtree(output_dir) + logger.info(f"Cleaned existing documentation in {output_dir}") - print(f"Generating API documentation...") - print(f"Output directory: {output_dir}") - print(f"Source directory: {source_dir}") + logger.info(f"Generating API documentation...") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Source directory: {source_dir}") - # Similar exclusions as in your generate_docs.py excludes = [ + str(project_root / "src/pipecat/pipeline/to_be_updated"), str(project_root / "src/pipecat/processors/gstreamer"), - str(project_root / "src/pipecat/transports/network"), - str(project_root / "src/pipecat/transports/services"), - str(project_root / "src/pipecat/transports/local"), str(project_root / "src/pipecat/services/to_be_updated"), + str(project_root / "src/pipecat/vad"), # deprecated "**/test_*.py", "**/tests/*.py", ] try: - main(["-f", "-e", "-M", "--no-toc", "-o", output_dir, source_dir] + excludes) - print("API documentation generated successfully!") + main( + [ + "-f", # Force overwriting + "-e", # Don't generate empty files + "-M", # Put module documentation before submodule documentation + "--no-toc", # Don't create a table of contents file + "--separate", # Put documentation for each module in its own page + "--module-first", # Module documentation before submodule documentation + "--implicit-namespaces", # Added: Handle implicit namespace packages + "-o", + output_dir, + source_dir, + ] + + excludes + ) + + logger.info("API documentation generated successfully!") + + # Process generated RST files to update titles + for rst_file in Path(output_dir).glob("**/*.rst"): # Changed to recursive glob + content = rst_file.read_text() + lines = content.split("\n") + + # Find and clean up the title + if lines and "=" in lines[1]: # Title is typically the first line + old_title = lines[0] + new_title = clean_title(old_title) + content = content.replace(old_title, new_title) + rst_file.write_text(content) + logger.info(f"Updated title: {old_title} -> {new_title}") + except Exception as e: - print(f"Error generating API documentation: {e}") + logger.error(f"Error generating API documentation: {e}", exc_info=True) + + +# Run module verification +verify_modules() diff --git a/docs/api/generate_docs.py b/docs/api/generate_docs.py deleted file mode 100644 index 972ea3c89..000000000 --- a/docs/api/generate_docs.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 - -import shutil -import subprocess -from pathlib import Path - - -def run_command(command: list[str]) -> None: - """Run a command and exit if it fails.""" - print(f"Running: {' '.join(command)}") - try: - subprocess.run(command, check=True) - except subprocess.CalledProcessError as e: - print(f"Warning: Command failed: {' '.join(command)}") - print(f"Error: {e}") - - -def main(): - docs_dir = Path(__file__).parent - project_root = docs_dir.parent.parent - - # Install documentation requirements - requirements_file = docs_dir / "requirements.txt" - run_command(["pip", "install", "-r", str(requirements_file)]) - - # Install from project root, not docs directory - run_command(["pip", "install", "-e", str(project_root)]) - - # Install all service dependencies - services = [ - "anthropic", - "assemblyai", - "aws", - "azure", - "canonical", - "cartesia", - # "daily", - "deepgram", - "elevenlabs", - "fal", - "fireworks", - "gladia", - "google", - "grok", - "groq", - "langchain", - # "livekit", - "lmnt", - "moondream", - "nim", - "noisereduce", - "openai", - "openpipe", - "playht", - "silero", - "soundfile", - "websocket", - "whisper", - ] - - extras = ",".join(services) - try: - run_command(["pip", "install", "-e", f"{str(project_root)}[{extras}]"]) - except Exception as e: - print(f"Warning: Some dependencies failed to install: {e}") - - # Clean old files - api_dir = docs_dir / "api" - build_dir = docs_dir / "_build" - for dir in [api_dir, build_dir]: - if dir.exists(): - shutil.rmtree(dir) - - # Generate API documentation - run_command( - [ - "sphinx-apidoc", - "-f", # Force overwrite - "-e", # Put each module on its own page - "-M", # Put module documentation before submodule - "--no-toc", # Don't generate modules.rst (cleaner structure) - "-o", - str(api_dir), # Output directory - str(project_root / "src/pipecat"), - # Exclude problematic files and directories - str(project_root / "src/pipecat/processors/gstreamer"), # Optional gstreamer - str(project_root / "src/pipecat/transports/network"), # Pydantic issues - str(project_root / "src/pipecat/transports/services"), # Pydantic issues - str(project_root / "src/pipecat/transports/local"), # Optional dependencies - str(project_root / "src/pipecat/services/to_be_updated"), # Exclude to_be_updated - "**/test_*.py", # Test files - "**/tests/*.py", # Test files - ] - ) - - # Build HTML documentation - run_command(["sphinx-build", "-b", "html", str(docs_dir), str(build_dir / "html")]) - - print("\nDocumentation generated successfully!") - print(f"HTML docs: {build_dir}/html/index.html") - - -if __name__ == "__main__": - main() diff --git a/docs/api/index.rst b/docs/api/index.rst index 68afcb980..ce7c22113 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -13,61 +13,61 @@ Quick Links * `GitHub Repository `_ * `Website `_ - API Reference ------------- Core Components ~~~~~~~~~~~~~~~ -* :mod:`pipecat.frames` -* :mod:`pipecat.processors` -* :mod:`pipecat.pipeline` +* :mod:`Frames ` +* :mod:`Processors ` +* :mod:`Pipeline ` Audio Processing ~~~~~~~~~~~~~~~~ -* :mod:`pipecat.audio` -* :mod:`pipecat.vad` +* :mod:`Audio ` Services ~~~~~~~~ -* :mod:`pipecat.services` +* :mod:`Services ` Transport & Serialization ~~~~~~~~~~~~~~~~~~~~~~~~~ -* :mod:`pipecat.transports` -* :mod:`pipecat.serializers` +* :mod:`Transports ` + * :mod:`Local ` + * :mod:`Network ` + * :mod:`Services ` +* :mod:`Serializers ` Utilities ~~~~~~~~~ -* :mod:`pipecat.clocks` -* :mod:`pipecat.metrics` -* :mod:`pipecat.sync` -* :mod:`pipecat.transcriptions` -* :mod:`pipecat.utils` +* :mod:`Clocks ` +* :mod:`Metrics ` +* :mod:`Sync ` +* :mod:`Transcriptions ` +* :mod:`Utils ` .. toctree:: - :maxdepth: 2 + :maxdepth: 3 :caption: API Reference :hidden: - api/pipecat.audio - api/pipecat.clocks - api/pipecat.frames - api/pipecat.metrics - api/pipecat.pipeline - api/pipecat.processors - api/pipecat.serializers - api/pipecat.services - api/pipecat.sync - api/pipecat.transcriptions - api/pipecat.transports - api/pipecat.utils - api/pipecat.vad + Audio + Clocks + Frames + Metrics + Pipeline + Processors + Serializers + Services + Sync + Transcriptions + Transports + Utils Indices and tables ================== diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt index f571b6ce0..afb4c8dcf 100644 --- a/docs/api/requirements.txt +++ b/docs/api/requirements.txt @@ -1,6 +1,40 @@ +# Sphinx dependencies sphinx>=8.1.3 sphinx-rtd-theme sphinx-markdown-builder sphinx-autodoc-typehints toml -pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,silero,soundfile,websocket,whisper] + +# Install all extras individually to ensure they're properly resolved +pipecat-ai[anthropic] +pipecat-ai[assemblyai] +pipecat-ai[aws] +pipecat-ai[azure] +pipecat-ai[canonical] +pipecat-ai[cartesia] +pipecat-ai[daily] +pipecat-ai[deepgram] +pipecat-ai[elevenlabs] +pipecat-ai[fal] +pipecat-ai[fireworks] +pipecat-ai[gladia] +pipecat-ai[google] +pipecat-ai[grok] +pipecat-ai[groq] +# pipecat-ai[krisp] # Mocked instead +pipecat-ai[langchain] +pipecat-ai[livekit] +pipecat-ai[lmnt] +pipecat-ai[local] +pipecat-ai[moondream] +pipecat-ai[nim] +pipecat-ai[noisereduce] +pipecat-ai[openai] +# pipecat-ai[openpipe] +# pipecat-ai[playht] # Mocked due to grpcio conflict with riva +pipecat-ai[riva] +pipecat-ai[silero] +pipecat-ai[simli] +pipecat-ai[soundfile] +pipecat-ai[websocket] +pipecat-ai[whisper] \ No newline at end of file diff --git a/docs/api/rtd-test.sh b/docs/api/rtd-test.sh new file mode 100755 index 000000000..2b2c30d5d --- /dev/null +++ b/docs/api/rtd-test.sh @@ -0,0 +1,38 @@ +#!/bin/bash +set -e + +# Configuration +DOCS_DIR=$(pwd) +PROJECT_ROOT=$(cd ../../ && pwd) +TEST_DIR="/tmp/rtd-test-$(date +%Y%m%d_%H%M%S)" + +echo "Creating test directory: $TEST_DIR" +mkdir -p "$TEST_DIR" +cd "$TEST_DIR" + +# Create virtual environment +python -m venv venv +source venv/bin/activate + +echo "Installing build dependencies..." +pip install --upgrade pip wheel setuptools + +echo "Installing documentation dependencies..." +pip install -r "$DOCS_DIR/requirements.txt" + +echo "Building documentation..." +cd "$DOCS_DIR" +sphinx-build -b html . "_build/html" + +echo "Build complete. Check _build/html directory for output." + +# Print summary +echo -e "\n=== Build Summary ===" +echo "Documentation: $DOCS_DIR/_build/html" +echo "Test environment: $TEST_DIR" +echo -e "\nTo view the documentation:" +echo "open $DOCS_DIR/_build/html/index.html" + +# Print installed packages for verification +echo -e "\n=== Installed Packages ===" +pip freeze | grep -E "sphinx|pipecat" \ No newline at end of file diff --git a/examples/canonical-metrics/bot.py b/examples/canonical-metrics/bot.py index 71aca70f3..2c2d35911 100644 --- a/examples/canonical-metrics/bot.py +++ b/examples/canonical-metrics/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/canonical-metrics/runner.py b/examples/canonical-metrics/runner.py index a0b46ca36..defe65dfd 100644 --- a/examples/canonical-metrics/runner.py +++ b/examples/canonical-metrics/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/canonical-metrics/server.py b/examples/canonical-metrics/server.py index 2ed6c8239..19cda92a3 100644 --- a/examples/canonical-metrics/server.py +++ b/examples/canonical-metrics/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/chatbot-audio-recording/bot.py b/examples/chatbot-audio-recording/bot.py index f020a9626..128ed3ec2 100644 --- a/examples/chatbot-audio-recording/bot.py +++ b/examples/chatbot-audio-recording/bot.py @@ -1,18 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiofiles import asyncio +import datetime import io import os import sys - -import aiohttp -import datetime import wave + +import aiofiles +import aiohttp from dotenv import load_dotenv from loguru import logger from runner import configure diff --git a/examples/chatbot-audio-recording/runner.py b/examples/chatbot-audio-recording/runner.py index a0b46ca36..defe65dfd 100644 --- a/examples/chatbot-audio-recording/runner.py +++ b/examples/chatbot-audio-recording/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/chatbot-audio-recording/server.py b/examples/chatbot-audio-recording/server.py index 2ed6c8239..19cda92a3 100644 --- a/examples/chatbot-audio-recording/server.py +++ b/examples/chatbot-audio-recording/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/deployment/flyio-example/bot.py b/examples/deployment/flyio-example/bot.py index 7c69f62bd..05e55016f 100644 --- a/examples/deployment/flyio-example/bot.py +++ b/examples/deployment/flyio-example/bot.py @@ -1,22 +1,21 @@ +import argparse import asyncio import os import sys -import argparse + +from dotenv import load_dotenv +from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame, EndFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.openai import OpenAILLMService from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/deployment/flyio-example/bot_runner.py b/examples/deployment/flyio-example/bot_runner.py index 3795847e4..d04bcd60f 100644 --- a/examples/deployment/flyio-example/bot_runner.py +++ b/examples/deployment/flyio-example/bot_runner.py @@ -1,29 +1,27 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse -import subprocess import os - +import subprocess from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from pipecat.transports.services.helpers.daily_rest import ( DailyRESTHelper, DailyRoomObject, - DailyRoomProperties, DailyRoomParams, + DailyRoomProperties, ) -from dotenv import load_dotenv - load_dotenv(override=True) diff --git a/examples/deployment/modal-example/app.py b/examples/deployment/modal-example/app.py index 97a9f32ab..21d85f79f 100644 --- a/examples/deployment/modal-example/app.py +++ b/examples/deployment/modal-example/app.py @@ -2,12 +2,11 @@ import os import aiohttp import modal +from bot import _voice_bot_process from fastapi import HTTPException from fastapi.responses import JSONResponse from loguru import logger -from bot import _voice_bot_process - MAX_SESSION_TIME = 15 * 60 # 15 minutes app = modal.App("pipecat-modal") diff --git a/examples/dialin-chatbot/bot_daily.py b/examples/dialin-chatbot/bot_daily.py index 79e8665b3..d277011ab 100644 --- a/examples/dialin-chatbot/bot_daily.py +++ b/examples/dialin-chatbot/bot_daily.py @@ -1,21 +1,20 @@ +import argparse import asyncio import os import sys -import argparse + +from dotenv import load_dotenv +from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame, EndFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport, DailyDialinSettings - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyDialinSettings, DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/dialin-chatbot/bot_runner.py b/examples/dialin-chatbot/bot_runner.py index ac2ce79bf..050ad62b9 100644 --- a/examples/dialin-chatbot/bot_runner.py +++ b/examples/dialin-chatbot/bot_runner.py @@ -7,14 +7,14 @@ provisioning a room and starting a Pipecat bot in response. Refer to README for more information. """ -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, PlainTextResponse from twilio.twiml.voice_response import VoiceResponse @@ -22,13 +22,11 @@ from twilio.twiml.voice_response import VoiceResponse from pipecat.transports.services.helpers.daily_rest import ( DailyRESTHelper, DailyRoomObject, + DailyRoomParams, DailyRoomProperties, DailyRoomSipParams, - DailyRoomParams, ) -from dotenv import load_dotenv - load_dotenv(override=True) diff --git a/examples/dialin-chatbot/bot_twilio.py b/examples/dialin-chatbot/bot_twilio.py index 135fcf4ea..86b37381a 100644 --- a/examples/dialin-chatbot/bot_twilio.py +++ b/examples/dialin-chatbot/bot_twilio.py @@ -1,24 +1,22 @@ +import argparse import asyncio import os import sys -import argparse + +from dotenv import load_dotenv +from loguru import logger +from twilio.rest import Client from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame, EndFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from twilio.rest import Client - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index f0fec28d8..14a89340b 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -1,26 +1,24 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys -from pipecat.frames.frames import EndFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.task import PipelineTask -from pipecat.pipeline.runner import PipelineRunner -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport - +import aiohttp +from dotenv import load_dotenv +from loguru import logger from runner import configure -from loguru import logger - -from dotenv import load_dotenv +from pipecat.frames.frames import EndFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index e62ab1020..7262f3cfe 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -1,14 +1,16 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,10 +19,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.audio import LocalAudioTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index 9b3efc603..ed610e027 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -4,6 +4,9 @@ import os import sys import aiohttp +from dotenv import load_dotenv +from livekit import api +from loguru import logger from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline @@ -12,12 +15,6 @@ from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.livekit import LiveKitParams, LiveKitTransport -from livekit import api - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/01c-fastpitch.py b/examples/foundational/01c-fastpitch.py index b041e6c7b..c9ccb2695 100644 --- a/examples/foundational/01c-fastpitch.py +++ b/examples/foundational/01c-fastpitch.py @@ -1,26 +1,24 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys -from pipecat.frames.frames import EndFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.task import PipelineTask -from pipecat.pipeline.runner import PipelineRunner -from pipecat.services.riva import FastPitchTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport - +import aiohttp +from dotenv import load_dotenv +from loguru import logger from runner import configure -from loguru import logger - -from dotenv import load_dotenv +from pipecat.frames.frames import EndFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.services.riva import FastPitchTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index ba2be607c..ec1209fa7 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,12 +21,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/03-still-frame.py b/examples/foundational/03-still-frame.py index 688146cb1..310953b1f 100644 --- a/examples/foundational/03-still-frame.py +++ b/examples/foundational/03-still-frame.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import EndFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -16,12 +20,6 @@ from pipecat.pipeline.task import PipelineTask from pipecat.services.fal import FalImageGenService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/03a-local-still-frame.py b/examples/foundational/03a-local-still-frame.py index c06834d90..4544e8de5 100644 --- a/examples/foundational/03a-local-still-frame.py +++ b/examples/foundational/03a-local-still-frame.py @@ -1,16 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys - import tkinter as tk +import aiohttp +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -19,10 +21,6 @@ from pipecat.services.fal import FalImageGenService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.tk import TkLocalTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/04-utterance-and-speech.py b/examples/foundational/04-utterance-and-speech.py index 7f63757d6..89f941838 100644 --- a/examples/foundational/04-utterance-and-speech.py +++ b/examples/foundational/04-utterance-and-speech.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,27 +8,24 @@ # This example broken on latest pipecat and needs updating. # -import aiohttp import asyncio import os import sys -from pipecat.pipeline.merge_pipeline import SequentialMergePipeline -from pipecat.pipeline.pipeline import Pipeline +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame +from pipecat.pipeline.merge_pipeline import SequentialMergePipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.task import PipelineTask from pipecat.services.azure import AzureLLMService, AzureTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.transport_services import TransportServiceOutput from pipecat.services.transports.daily_transport import DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 64f85930b..7990df1c8 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -1,16 +1,19 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys - from dataclasses import dataclass +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import ( DataFrame, Frame, @@ -22,19 +25,13 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.sync_parallel_pipeline import SyncParallelPipeline from pipecat.pipeline.task import PipelineTask -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia import CartesiaHttpTTSService -from pipecat.services.openai import OpenAILLMService from pipecat.services.fal import FalImageGenService +from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 4a561c073..e29a20f32 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -1,23 +1,25 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - import tkinter as tk +import aiohttp +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import ( Frame, + LLMMessagesFrame, OutputAudioRawFrame, + TextFrame, TTSAudioRawFrame, URLImageRawFrame, - LLMMessagesFrame, - TextFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -26,15 +28,11 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.sentence import SentenceAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia import CartesiaHttpTTSService -from pipecat.services.openai import OpenAILLMService from pipecat.services.fal import FalImageGenService +from pipecat.services.openai import OpenAILLMService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.tk import TkLocalTransport, TkOutputTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 5f9bb0a1f..4ccca2792 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index eda3c61df..853dacf5b 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -1,15 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger from PIL import Image +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, OutputImageRawFrame, SystemFrame, TextFrame @@ -20,14 +23,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia import CartesiaHttpTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import DailyTransport - -from pipecat.transports.services.daily import DailyParams -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index 59013bd3a..d496b582f 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -1,30 +1,28 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.audio.vad.silero import SileroVAD from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.audio.vad.silero import SileroVAD from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 3148986a8..94fe43c23 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,12 +23,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index e7e680eab..52e1bf2fa 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index e86c4a8e9..3b596cd24 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -9,6 +9,14 @@ import os import sys import aiohttp +from dotenv import load_dotenv +from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_community.chat_message_histories import ChatMessageHistory +from langchain_core.chat_history import BaseChatMessageHistory +from langchain_core.runnables.history import RunnableWithMessageHistory +from langchain_openai import ChatOpenAI +from loguru import logger +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -23,18 +31,6 @@ from pipecat.processors.frameworks.langchain import LangchainProcessor from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder -from langchain_community.chat_message_histories import ChatMessageHistory -from langchain_core.chat_history import BaseChatMessageHistory -from langchain_core.runnables.history import RunnableWithMessageHistory -from langchain_openai import ChatOpenAI - -from loguru import logger - -from runner import configure - -from dotenv import load_dotenv - load_dotenv(override=True) diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index 40866d760..18546759c 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index e52ca370e..0af28ed41 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index 24ccf6a24..ff950acd5 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -11,7 +11,6 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,6 +18,7 @@ from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 293e0b74d..bb3febe68 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index fb3e8ea0c..89ff5b7eb 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -18,13 +22,6 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.azure import AzureLLMService, AzureSTTService, AzureTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07g-interruptible-openai-tts.py b/examples/foundational/07g-interruptible-openai-tts.py index af414e3e3..32d537229 100644 --- a/examples/foundational/07g-interruptible-openai-tts.py +++ b/examples/foundational/07g-interruptible-openai-tts.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -11,7 +11,6 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,6 +18,7 @@ from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService, OpenAITTSService from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index f01d535df..deb606e7d 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -1,13 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import time + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -19,13 +24,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openpipe import OpenPipeLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger -import time - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 5bfe13b5c..28ba557f7 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,12 +23,6 @@ from pipecat.services.openai import OpenAILLMService from pipecat.services.xtts import XTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index b6866d8af..e2ad5178d 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index 49179e747..683bb493f 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,12 +23,6 @@ from pipecat.services.lmnt import LmntTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py index 96315623b..3de029c3e 100644 --- a/examples/foundational/07l-interruptible-together.py +++ b/examples/foundational/07l-interruptible-together.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07m-interruptible-polly.py b/examples/foundational/07m-interruptible-polly.py index df376b2f4..22e3059c8 100644 --- a/examples/foundational/07m-interruptible-polly.py +++ b/examples/foundational/07m-interruptible-polly.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index c0b29715e..b6322d2b9 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 0ae548f9c..49f18aa5e 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 834e59037..dfc085f8f 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -13,6 +13,7 @@ from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.audio.filters.krisp_filter import KrispFilter from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -25,7 +26,6 @@ from pipecat.services.deepgram import DeepgramSTTService, DeepgramTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer -from pipecat.audio.filters.krisp_filter import KrispFilter load_dotenv(override=True) diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 2e13e2a8e..7f13be30a 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index 1e02f82c0..2a8fcce7a 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 1778e0c62..cd250ba07 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -1,41 +1,40 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - -import google.ai.generativelanguage as glm - from dataclasses import dataclass + +import aiohttp +import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.google import GoogleLLMService -from pipecat.processors.frame_processor import FrameProcessor -from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.frames.frames import ( - LLMFullResponseStartFrame, - LLMFullResponseEndFrame, - InputAudioRawFrame, Frame, + InputAudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, StartInterruptionFrame, TextFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.google import GoogleLLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py new file mode 100644 index 000000000..c09f67a29 --- /dev/null +++ b/examples/foundational/07t-interruptible-fish.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.fish import FishAudioTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = FishAudioTTSService( + api_key=os.getenv("FISH_API_KEY"), + model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py index 150fbfc0a..e455b321b 100644 --- a/examples/foundational/08-bots-arguing.py +++ b/examples/foundational/08-bots-arguing.py @@ -1,20 +1,19 @@ -from typing import Tuple -import aiohttp import asyncio import logging import os -from pipecat.processors.aggregators import SentenceAggregator -from pipecat.pipeline.pipeline import Pipeline +from typing import Tuple -from pipecat.transports.services.daily import DailyTransport +import aiohttp +from dotenv import load_dotenv +from runner import configure + +from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators import SentenceAggregator from pipecat.services.azure import AzureLLMService, AzureTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.fal import FalImageGenService -from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame - -from runner import configure - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index a719d54f6..f5a1cf337 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -1,13 +1,17 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -19,13 +23,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.transports.services.daily import DailyTransport, DailyParams - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index 539cca600..2ad41ef5f 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -1,15 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import sys - import tkinter as tk +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -25,12 +28,6 @@ from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.tk import TkLocalTransport from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index 982303aee..e943238d3 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -19,12 +23,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index d8692a7f1..9bce4769f 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -1,15 +1,19 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys import wave +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( Frame, @@ -29,12 +33,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index b5bb577aa..0cfd6a729 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.moondream import MoondreamService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index bc76afc73..64e168b62 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.google import GoogleLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index d8474b568..22a303fee 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index bc6f5a4ea..06c35e3ae 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -17,16 +21,10 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/13-whisper-transcription.py b/examples/foundational/13-whisper-transcription.py index c895cb944..7ca11522d 100644 --- a/examples/foundational/13-whisper-transcription.py +++ b/examples/foundational/13-whisper-transcription.py @@ -1,13 +1,17 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -16,12 +20,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper import WhisperSTTService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/13a-whisper-local.py b/examples/foundational/13a-whisper-local.py index c1ba37ca9..88ec65149 100644 --- a/examples/foundational/13a-whisper-local.py +++ b/examples/foundational/13a-whisper-local.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -7,6 +7,9 @@ import asyncio import sys +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -16,10 +19,6 @@ from pipecat.services.whisper import WhisperSTTService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.audio import LocalAudioTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/13b-deepgram-transcription.py b/examples/foundational/13b-deepgram-transcription.py index 7b3a25316..e7a6ca335 100644 --- a/examples/foundational/13b-deepgram-transcription.py +++ b/examples/foundational/13b-deepgram-transcription.py @@ -1,28 +1,26 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.deepgram import DeepgramSTTService, LiveOptions, Language +from pipecat.services.deepgram import DeepgramSTTService, Language, LiveOptions from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/13c-gladia-transcription.py b/examples/foundational/13c-gladia-transcription.py index acc21b6c2..06488f094 100644 --- a/examples/foundational/13c-gladia-transcription.py +++ b/examples/foundational/13c-gladia-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/13d-assemblyai-transcription.py b/examples/foundational/13d-assemblyai-transcription.py index d10a80274..db7dba57b 100644 --- a/examples/foundational/13d-assemblyai-transcription.py +++ b/examples/foundational/13d-assemblyai-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 2479e3f3e..bccc18196 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -1,14 +1,19 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,14 +22,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 9d3335809..04eb75cbb 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -1,29 +1,27 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index cdc69556d..689d6935b 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -1,29 +1,27 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index a629753e5..fc6836147 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index e238d91eb..ce0298ca9 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -1,14 +1,19 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,14 +22,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index ede7222bb..34748f39e 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -18,12 +22,6 @@ from pipecat.services.google import GoogleLLMService from pipecat.services.openai import OpenAILLMContext from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index bcd26672f..8140dc9c4 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 0165f1f7e..d83288c24 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 29b9ef15a..eb9f3b364 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index b3b10df87..462f0107a 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 68bb3b52e..89e059a5b 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -65,7 +65,7 @@ async def main(): ) llm = NimLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct" + api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct" ) # Register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. @@ -76,18 +76,18 @@ async def main(): type="function", function={ "name": "get_current_weather", - "description": "Get the current weather", + "description": "Returns the current weather at a location, if one is specified, and defaults to the user's location.", "parameters": { "type": "object", "properties": { "location": { "type": "string", - "description": "The city and state, e.g. San Francisco, CA", + "description": "The location to find the weather of, or if not provided, it's the default location.", }, "format": { "type": "string", "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", + "description": "Whether to use SI or USCS units (celsius or fahrenheit).", }, }, "required": ["location", "format"], diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py new file mode 100644 index 000000000..d41266105 --- /dev/null +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.cerebras import CerebrasLLMService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def start_fetch_weather(function_name, llm, context): + # note: we can't push a frame to the LLM here. the bot + # can interrupt itself and/or cause audio overlapping glitches. + # possible question for Aleix and Chad about what the right way + # to trigger speech is, now, with the new queues/async/sync refactors. + # await llm.push_frame(TextFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather for a specific location. You MUST use this function whenever asked about weather.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Use fahrenheit for US locations, celsius for others.", + }, + }, + "required": ["location", "format"], + }, + }, + ) + ] + messages = [ + { + "role": "system", + "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. + +You have one functions available: + +1. get_current_weather is used to get current weather information. + +Infer whether to use Fahrenheit or Celsius automatically based on the location, unless the user specifies a preference. + +Start by asking me for my location. Then, use 'get_weather_current' to give me a forecast. + + Respond to what the user said in a creative and helpful way.""", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 5c61dd25f..8e1a7ed16 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -1,18 +1,23 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame -from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -21,14 +26,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index 4b8794ab8..0e9dc2948 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -1,20 +1,24 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp from deepgram import LiveOptions +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame -from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -24,14 +28,6 @@ from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index 85a2ddb0b..686aa01ac 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 7268e2305..d0e6eca27 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -20,12 +24,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/18-gstreamer-filesrc.py b/examples/foundational/18-gstreamer-filesrc.py index 8ebcaa1b9..c76d8f5b0 100644 --- a/examples/foundational/18-gstreamer-filesrc.py +++ b/examples/foundational/18-gstreamer-filesrc.py @@ -1,26 +1,24 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio -import aiohttp import argparse +import asyncio import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure_with_args + from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure_with_args - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/18a-gstreamer-videotestsrc.py b/examples/foundational/18a-gstreamer-videotestsrc.py index 9e5977348..5f91a3a3f 100644 --- a/examples/foundational/18a-gstreamer-videotestsrc.py +++ b/examples/foundational/18a-gstreamer-videotestsrc.py @@ -1,25 +1,23 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 49ede65be..383160f27 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index d4d418e91..0c6cd49be 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -24,9 +24,8 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) -from pipecat.services.openai import OpenAILLMService from pipecat.services.cartesia import CartesiaTTSService - +from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index c22eaf128..7398891b0 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 421b00603..9bbcad278 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -24,9 +24,8 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService - +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 25d809021..faef2b429 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -26,7 +26,6 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.google import GoogleLLMService - from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/21-tavus-layer.py b/examples/foundational/21-tavus-layer.py index 61705b28a..f985bfba1 100644 --- a/examples/foundational/21-tavus-layer.py +++ b/examples/foundational/21-tavus-layer.py @@ -1,16 +1,19 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys - from typing import Any, Mapping +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -20,15 +23,10 @@ from pipecat.processors.aggregators.llm_response import ( LLMUserResponseAggregator, ) from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.openai import OpenAILLMService from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService from pipecat.services.tavus import TavusVideoService from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.audio.vad.silero import SileroVADAnalyzer - -from loguru import logger - -from dotenv import load_dotenv load_dotenv(override=True) diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 73dfb003d..dc47e0956 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -1,18 +1,22 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.gated_openai_llm_context import GatedOpenAILLMContextAggregator @@ -26,12 +30,6 @@ from pipecat.services.openai import OpenAILLMService from pipecat.sync.event_notifier import EventNotifier from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 2deeb3da4..92dd98d4b 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -1,53 +1,51 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys import time +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.services.openai import OpenAILLMService -from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.processors.frame_processor import FrameProcessor, FrameDirection from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, + LLMMessagesFrame, StartFrame, StartInterruptionFrame, StopInterruptionFrame, SystemFrame, + TextFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame -from pipecat.sync.base_notifier import BaseNotifier +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.processors.filters.function_filter import FunctionFilter +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.user_idle_processor import UserIdleProcessor - - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.sync.base_notifier import BaseNotifier +from pipecat.sync.event_notifier import EventNotifier +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 97bc57ec1..8a3d60c76 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 1ff8aa23e..4d618f726 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -1,93 +1,377 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys import time +import aiohttp +import google.ai.generativelanguage as glm +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.google import GoogleLLMService, GoogleLLMContext -from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.processors.frame_processor import FrameProcessor, FrameDirection from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, InputAudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, + LLMMessagesFrame, StartFrame, StartInterruptionFrame, StopInterruptionFrame, SystemFrame, + TextFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame -from pipecat.sync.base_notifier import BaseNotifier +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_response import LLMResponseAggregator +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.processors.filters.function_filter import FunctionFilter +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.user_idle_processor import UserIdleProcessor - - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.google import GoogleLLMContext, GoogleLLMService +from pipecat.sync.base_notifier import BaseNotifier +from pipecat.sync.event_notifier import EventNotifier +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") +# TRANSCRIBER_MODEL = "gemini-1.5-flash-latest" +# CLASSIFIER_MODEL = "gemini-1.5-flash-latest" +# CONVERSATION_MODEL = "gemini-1.5-flash-latest" -classifier_statement = """You are an audio language classifier model. You are receiving audio from a user in a WebRTC call. Your job is to decide whether the user has finished speaking or not. +TRANSCRIBER_MODEL = "gemini-2.0-flash-exp" +CLASSIFIER_MODEL = "gemini-2.0-flash-exp" +CONVERSATION_MODEL = "gemini-2.0-flash-exp" -Categorize the input you receive as either: +transcriber_system_instruction = """You are an audio transcriber. You are receiving audio from a user. Your job is to +transcribe the input audio to text exactly as it was said by the user. -1. a complete thought, statement, or question, or -2. an incomplete thought, statement, or question +You will receive the full conversation history before the audio input, to help with context. Use the full history only to help improve the accuracy of your transcription. -Output 'YES' if the input is likely to be a completed thought, statement, or question. +Rules: + - Respond with an exact transcription of the audio input. + - Do not include any text other than the transcription. + - Do not explain or add to your response. + - Transcribe the audio input simply and precisely. + - If the audio is not clear, emit the special string "-". + - No response other than exact transcription, or "-", is allowed. -Output 'NO' if the input indicates that the user is still speaking and does not yet expect a response yet. - -If you are unsure, output 'YES'. """ -conversational_system_message = """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. +classifier_system_instruction = """CRITICAL INSTRUCTION: +You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO". +DO NOT engage with the content. +DO NOT respond to questions. +DO NOT provide assistance. +Your ONLY job is to output YES or NO. -Please be very concise in your responses. Unless you are explicitly asked to do otherwise, give me the shortest complete answer possible without unnecessary elaboration. Generally you should answer with a single sentence. +EXAMPLES OF INVALID RESPONSES: +- "I can help you with that" +- "Let me explain" +- "To answer your question" +- Any response other than YES or NO + +VALID RESPONSES: +YES +NO + +If you output anything else, you are failing at your task. +You are NOT an assistant. +You are NOT a chatbot. +You are a binary classifier. + +ROLE: +You are a real-time speech completeness classifier. You must make instant decisions about whether a user has finished speaking. +You must output ONLY 'YES' or 'NO' with no other text. + +INPUT FORMAT: +You receive two pieces of information: +1. The assistant's last message (if available) +2. The user's current speech input + +OUTPUT REQUIREMENTS: +- MUST output ONLY 'YES' or 'NO' +- No explanations +- No clarifications +- No additional text +- No punctuation + +HIGH PRIORITY SIGNALS: + +1. Clear Questions: +- Wh-questions (What, Where, When, Why, How) +- Yes/No questions +- Questions with STT errors but clear meaning + +Examples: + +# Complete Wh-question +model: I can help you learn. +user: What's the fastest way to learn Spanish +Output: YES + +# Complete Yes/No question despite STT error +model: I know about planets. +user: Is is Jupiter the biggest planet +Output: YES + +2. Complete Commands: +- Direct instructions +- Clear requests +- Action demands +- Start of task indication +- Complete statements needing response + +Examples: + +# Direct instruction +model: I can explain many topics. +user: Tell me about black holes +Output: YES + +# Start of task indication +user: Let's begin. +Output: YES + +# Start of task indication +user: Let's get started. +Output: YES + +# Action demand +model: I can help with math. +user: Solve this equation x plus 5 equals 12 +Output: YES + +3. Direct Responses: +- Answers to specific questions +- Option selections +- Clear acknowledgments with completion +- Providing information with a known format - mailing address +- Providing information with a known format - phone number +- Providing information with a known format - credit card number + +Examples: + +# Specific answer +model: What's your favorite color? +user: I really like blue +Output: YES + +# Option selection +model: Would you prefer morning or evening? +user: Morning +Output: YES + +# Providing information with a known format - mailing address +model: What's your address? +user: 1234 Main Street +Output: NO + +# Providing information with a known format - mailing address +model: What's your address? +user: 1234 Main Street Irving Texas 75063 +Output: Yes + +# Providing information with a known format - phone number +model: What's your phone number? +user: 41086753 +Output: NO + +# Providing information with a known format - phone number +model: What's your phone number? +user: 4108675309 +Output: Yes + +# Providing information with a known format - phone number +model: What's your phone number? +user: 220 +Output: No + +# Providing information with a known format - credit card number +model: What's your credit card number? +user: 5556 +Output: NO + +# Providing information with a known format - phone number +model: What's your credit card number? +user: 5556710454680800 +Output: Yes + +model: What's your credit card number? +user: 414067 +Output: NO + + +MEDIUM PRIORITY SIGNALS: + +1. Speech Pattern Completions: +- Self-corrections reaching completion +- False starts with clear ending +- Topic changes with complete thought +- Mid-sentence completions + +Examples: + +# Self-correction reaching completion +model: What would you like to know? +user: Tell me about... no wait, explain how rainbows form +Output: YES + +# Topic change with complete thought +model: The weather is nice today. +user: Actually can you tell me who invented the telephone +Output: YES + +# Mid-sentence completion +model: Hello I'm ready. +user: What's the capital of? France +Output: YES + +2. Context-Dependent Brief Responses: +- Acknowledgments (okay, sure, alright) +- Agreements (yes, yeah) +- Disagreements (no, nah) +- Confirmations (correct, exactly) + +Examples: + +# Acknowledgment +model: Should we talk about history? +user: Sure +Output: YES + +# Disagreement with completion +model: Is that what you meant? +user: No not really +Output: YES + +LOW PRIORITY SIGNALS: + +1. STT Artifacts (Consider but don't over-weight): +- Repeated words +- Unusual punctuation +- Capitalization errors +- Word insertions/deletions + +Examples: + +# Word repetition but complete +model: I can help with that. +user: What what is the time right now +Output: YES + +# Missing punctuation but complete +model: I can explain that. +user: Please tell me how computers work +Output: YES + +2. Speech Features: +- Filler words (um, uh, like) +- Thinking pauses +- Word repetitions +- Brief hesitations + +Examples: + +# Filler words but complete +model: What would you like to know? +user: Um uh how do airplanes fly +Output: YES + +# Thinking pause but incomplete +model: I can explain anything. +user: Well um I want to know about the +Output: NO + +DECISION RULES: + +1. Return YES if: +- ANY high priority signal shows clear completion +- Medium priority signals combine to show completion +- Meaning is clear despite low priority artifacts + +2. Return NO if: +- No high priority signals present +- Thought clearly trails off +- Multiple incomplete indicators +- User appears mid-formulation + +3. When uncertain: +- If you can understand the intent → YES +- If meaning is unclear → NO +- Always make a binary decision +- Never request clarification + +Examples: + +# Incomplete despite corrections +model: What would you like to know about? +user: Can you tell me about +Output: NO + +# Complete despite multiple artifacts +model: I can help you learn. +user: How do you I mean what's the best way to learn programming +Output: YES + +# Trailing off incomplete +model: I can explain anything. +user: I was wondering if you could tell me why +Output: NO +""" + +conversation_system_instruction = """You are a helpful assistant participating in a voice converation. + +Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. + +If you know that a number string is a phone number from the context of the conversation, write it as a phone number. For example 210-333-4567. + +If you know that a number string is a credit card number, write it as a credit card number. For example 4111-1111-1111-1111. + +Please be very concise in your responses. Unless you are explicitly asked to do otherwise, give me shortest complete answer possible without unnecessary elaboration. Generally you should answer with a single sentence. """ -class StatementJudgeAudioContextAccumulator(FrameProcessor): - def __init__(self, *, notifier: BaseNotifier, **kwargs): +class AudioAccumulator(FrameProcessor): + """Buffers user audio until the user stops speaking. + + Always pushes a fresh context with a single audio message. + """ + + def __init__(self, **kwargs): super().__init__(**kwargs) - self._notifier = notifier - self._audio_frames = [] self._audio_frames = [] self._start_secs = 0.2 # this should match VAD start_secs (hardcoding for now) - self._user_speaking = False + self._max_buffer_size_secs = 30 + self._user_speaking_vad_state = False + self._user_speaking_utterance_state = False async def reset(self): self._audio_frames = [] - self._user_speaking = False + self._user_speaking_vad_state = False + self._user_speaking_utterance_state = False async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -101,22 +385,33 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): # but let's leave that as an exercise to the reader. :-) return if isinstance(frame, UserStartedSpeakingFrame): - self._user_speaking = True + self._user_speaking_vad_state = True + self._user_speaking_utterance_state = True + elif isinstance(frame, UserStoppedSpeakingFrame): + data = b"".join(frame.audio for frame in self._audio_frames) + logger.debug( + f"Processing audio buffer seconds: ({len(self._audio_frames)}) ({len(data)}) {len(data) / 2 / 16000}" + ) self._user_speaking = False context = GoogleLLMContext() - context.set_messages([{"role": "system", "content": classifier_statement}]) - context.add_audio_frames_message(audio_frames=self._audio_frames) + context.add_audio_frames_message(text="Audio follows", audio_frames=self._audio_frames) await self.push_frame(OpenAILLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): - if self._user_speaking: - self._audio_frames.append(frame) + # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest + # frames as necessary. + # Use a small buffer size when an utterance is not in progress. Just big enough to backfill the start_secs. + # Use a larger buffer size when an utterance is in progress. + # Assume all audio frames have the same duration. + self._audio_frames.append(frame) + frame_duration = len(frame.audio) / 2 * frame.num_channels / frame.sample_rate + buffer_duration = frame_duration * len(self._audio_frames) + # logger.debug(f"!!! Frame duration: {frame_duration}") + if self._user_speaking_utterance_state: + while buffer_duration > self._max_buffer_size_secs: + self._audio_frames.pop(0) + buffer_duration -= frame_duration else: - # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest - # frames as necessary. Assume all audio frames have the same duration. - self._audio_frames.append(frame) - frame_duration = len(frame.audio) / 16 * frame.num_channels / frame.sample_rate - buffer_duration = frame_duration * len(self._audio_frames) while buffer_duration > self._start_secs: self._audio_frames.pop(0) buffer_duration -= frame_duration @@ -125,32 +420,143 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): class CompletenessCheck(FrameProcessor): - def __init__( - self, notifier: BaseNotifier, audio_accumulator: StatementJudgeAudioContextAccumulator - ): + """Checks the result of the classifier LLM to determine if the user has finished speaking. + + Triggers the notifier if the user has finished speaking. Also triggers the notifier if an + idle timeout is reached. + """ + + wait_time = 5.0 + + def __init__(self, notifier: BaseNotifier, audio_accumulator: AudioAccumulator, **kwargs): super().__init__() self._notifier = notifier self._audio_accumulator = audio_accumulator + self._idle_task = None + self._wakeup_time = 0 async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - if isinstance(frame, TextFrame) and frame.text.startswith("YES"): + if isinstance(frame, UserStartedSpeakingFrame): + if self._idle_task: + self._idle_task.cancel() + elif isinstance(frame, TextFrame) and frame.text.startswith("YES"): logger.debug("Completeness check YES") + if self._idle_task: + self._idle_task.cancel() await self.push_frame(UserStoppedSpeakingFrame()) await self._audio_accumulator.reset() await self._notifier.notify() elif isinstance(frame, TextFrame): if frame.text.strip(): logger.debug(f"Completeness check NO - '{frame.text}'") + # start timer to wake up if necessary + if self._wakeup_time: + self._wakeup_time = time.time() + self.wait_time + else: + # logger.debug("!!! CompletenessCheck idle wait START") + self._wakeup_time = time.time() + self.wait_time + self._idle_task = self.get_event_loop().create_task(self._idle_task_handler()) + + async def _idle_task_handler(self): + try: + while time.time() < self._wakeup_time: + await asyncio.sleep(0.01) + # logger.debug(f"!!! CompletenessCheck idle wait OVER") + await self._audio_accumulator.reset() + await self._notifier.notify() + except asyncio.CancelledError: + # logger.debug(f"!!! CompletenessCheck idle wait CANCEL") + pass + except Exception as e: + logger.error(f"CompletenessCheck idle wait error: {e}") + raise e + finally: + # logger.debug(f"!!! CompletenessCheck idle wait FINALLY") + self._wakeup_time = 0 + self._idle_task = None + + +class UserAggregatorBuffer(LLMResponseAggregator): + """Buffers the output of the transcription LLM. Used by the bot output gate.""" + + def __init__(self, **kwargs): + super().__init__( + messages=None, + role=None, + start_frame=LLMFullResponseStartFrame, + end_frame=LLMFullResponseEndFrame, + accumulator_frame=TextFrame, + handle_interruptions=True, + expect_stripped_words=False, + ) + self._transcription = "" + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # parent method pushes frames + if isinstance(frame, UserStartedSpeakingFrame): + self._transcription = "" + + async def _push_aggregation(self): + if self._aggregation: + self._transcription = self._aggregation + self._aggregation = "" + + logger.debug(f"[Transcription] {self._transcription}") + + async def wait_for_transcription(self): + while not self._transcription: + await asyncio.sleep(0.01) + tx = self._transcription + self._transcription = "" + return tx + + +class ConversationAudioContextAssembler(FrameProcessor): + """Takes the single-message context generated by the AudioAccumulator and adds it to the conversation LLM's context.""" + + def __init__(self, context: OpenAILLMContext, **kwargs): + super().__init__(**kwargs) + self._context = context + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # We must not block system frames. + if isinstance(frame, SystemFrame): + await self.push_frame(frame, direction) + return + + if isinstance(frame, OpenAILLMContextFrame): + GoogleLLMContext.upgrade_to_google(self._context) + last_message = frame.context.messages[-1] + self._context._messages.append(last_message) + await self.push_frame(OpenAILLMContextFrame(context=self._context)) class OutputGate(FrameProcessor): - def __init__(self, notifier: BaseNotifier, **kwargs): + """Buffers output frames until the notifier is triggered. + + When the notifier fires, waits until a transcription is ready, then: + 1. Replaces the last user audio message with the transcription. + 2. Flushes the frames buffer. + """ + + def __init__( + self, + notifier: BaseNotifier, + context: OpenAILLMContext, + user_transcription_buffer: "UserAggregatorBuffer", + **kwargs, + ): super().__init__(**kwargs) self._gate_open = False self._frames_buffer = [] self._notifier = notifier + self._context = context + self._transcription_buffer = user_transcription_buffer def close_gate(self): self._gate_open = False @@ -178,6 +584,13 @@ class OutputGate(FrameProcessor): await self.push_frame(frame, direction) return + if isinstance(frame, LLMFullResponseStartFrame): + # Remove the audio message from the context. We will never need it again. + # If the completeness check fails, a new audio message will be appended to the context. + # If the completeness check succeeds, our notifier will fire and we will append the + # transcription to the context. + self._context._messages.pop() + if self._gate_open: await self.push_frame(frame, direction) return @@ -196,12 +609,22 @@ class OutputGate(FrameProcessor): while True: try: await self._notifier.wait() + + transcription = await self._transcription_buffer.wait_for_transcription() or "-" + self._context._messages.append( + glm.Content(role="user", parts=[glm.Part(text=transcription)]) + ) + self.open_gate() for frame, direction in self._frames_buffer: await self.push_frame(frame, direction) self._frames_buffer = [] except asyncio.CancelledError: break + except Exception as e: + logger.error(f"OutputGate error: {e}") + raise e + break async def main(): @@ -217,64 +640,63 @@ async def main(): vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), vad_audio_passthrough=True, + audio_in_sample_rate=16000, ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) - # This is the LLM that will be used to detect if the user has finished a - # statement. This doesn't really need to be an LLM, we could use NLP - # libraries for that, but we have the machinery to use an LLM, so we might as well! - statement_llm = GoogleLLMService( - model="gemini-1.5-flash-latest", api_key=os.getenv("GOOGLE_API_KEY") + # This is the LLM that will transcribe user speech. + tx_llm = GoogleLLMService( + name="Transcriber", + model=TRANSCRIBER_MODEL, + api_key=os.getenv("GOOGLE_API_KEY"), + temperature=0.0, + system_instruction=transcriber_system_instruction, ) - # This is the regular LLM. - llm = GoogleLLMService(model="gemini-1.5-flash-latest", api_key=os.getenv("GOOGLE_API_KEY")) + # This is the LLM that will classify user speech as complete or incomplete. + classifier_llm = GoogleLLMService( + name="Classifier", + model=CLASSIFIER_MODEL, + api_key=os.getenv("GOOGLE_API_KEY"), + temperature=0.0, + system_instruction=classifier_system_instruction, + ) - messages = [ - { - "role": "system", - "content": conversational_system_message, - }, - ] + # This is the regular LLM that responds conversationally. + conversation_llm = GoogleLLMService( + name="Conversation", + model=CONVERSATION_MODEL, + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=conversation_system_instruction, + ) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) + context = OpenAILLMContext() + context_aggregator = conversation_llm.create_context_aggregator(context) - # We have instructed the LLM to return 'YES' if it thinks the user - # completed a sentence. So, if it's 'YES' we will return true in this + # We have instructed the LLM to return 'True' if it thinks the user + # completed a sentence. So, if it's 'True' we will return true in this # predicate which will wake up the notifier. async def wake_check_filter(frame): - return frame.text == "YES" + return frame.text == "True" # This is a notifier that we use to synchronize the two LLMs. notifier = EventNotifier() # This turns the LLM context into an inference request to classify the user's speech # as complete or incomplete. - statement_judge_context_filter = StatementJudgeAudioContextAccumulator(notifier=notifier) + # statement_judge_context_filter = StatementJudgeAudioContextAccumulator(notifier=notifier) + audio_accumulater = AudioAccumulator() # This sends a UserStoppedSpeakingFrame and triggers the notifier event completeness_check = CompletenessCheck( - notifier=notifier, audio_accumulator=statement_judge_context_filter + notifier=notifier, audio_accumulator=audio_accumulater ) - # # Notify if the user hasn't said anything. - async def user_idle_notifier(frame): - await notifier.notify() - - # Sometimes the LLM will fail detecting if a user has completed a - # sentence, this will wake up the notifier if that happens. - user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=5.0) - - bot_output_gate = OutputGate(notifier=notifier) - async def block_user_stopped_speaking(frame): return not isinstance(frame, UserStoppedSpeakingFrame) @@ -286,9 +708,18 @@ async def main(): or isinstance(frame, StopInterruptionFrame) ) + conversation_audio_context_assembler = ConversationAudioContextAssembler(context=context) + + user_aggregator_buffer = UserAggregatorBuffer() + + bot_output_gate = OutputGate( + notifier=notifier, context=context, user_transcription_buffer=user_aggregator_buffer + ) + pipeline = Pipeline( [ transport.input(), + audio_accumulater, ParallelPipeline( [ # Pass everything except UserStoppedSpeaking to the elements after @@ -296,24 +727,28 @@ async def main(): FunctionFilter(filter=block_user_stopped_speaking), ], [ - statement_judge_context_filter, - statement_llm, - completeness_check, + ParallelPipeline( + [ + classifier_llm, + completeness_check, + ], + [ + tx_llm, + user_aggregator_buffer, + ], + ) ], [ - stt, - context_aggregator.user(), - # Block everything except OpenAILLMContextFrame and LLMMessagesFrame - FunctionFilter(filter=pass_only_llm_trigger_frames), - llm, - bot_output_gate, # Buffer all llm/tts output until notified. + conversation_audio_context_assembler, + conversation_llm, + bot_output_gate, # buffer output until notified, then flush frames and update context + # TempPrinter(), ], ), tts, - user_idle, transport.output(), context_aggregator.assistant(), - ] + ], ) task = PipelineTask( diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index 68dd700d5..a8809648c 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -1,18 +1,22 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import argparse import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure_with_args + from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, MixerUpdateSettingsFrame, MixerEnableFrame +from pipecat.frames.frames import LLMMessagesFrame, MixerEnableFrame, MixerUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure_with_args - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index c1cb33aed..cdd248850 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index 843d24e1f..472ca974b 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -1,34 +1,21 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - -import google.ai.generativelanguage as glm - from dataclasses import dataclass + +import aiohttp +import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.google import GoogleLLMService, GoogleLLMContext -from pipecat.processors.frame_processor import FrameProcessor -from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -40,6 +27,18 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.google import GoogleLLMContext, GoogleLLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -81,8 +80,7 @@ Rules: class UserAudioCollector(FrameProcessor): - """ - This FrameProcessor collects audio frames in a buffer, then adds them to the + """This FrameProcessor collects audio frames in a buffer, then adds them to the LLM context when the user stops speaking. """ @@ -126,8 +124,7 @@ class UserAudioCollector(FrameProcessor): class InputTranscriptionContextFilter(FrameProcessor): - """ - This FrameProcessor blocks all frames except the OpenAILLMContextFrame that triggers + """This FrameProcessor blocks all frames except the OpenAILLMContextFrame that triggers LLM inference. (And system frames, which are needed for the pipeline element lifecycle.) We take the context object out of the OpenAILLMContextFrame and use it to create a new @@ -187,8 +184,7 @@ class InputTranscriptionContextFilter(FrameProcessor): @dataclass class LLMDemoTranscriptionFrame(Frame): - """ - It would be nice if we could just use a TranscriptionFrame to send our transcriber + """It would be nice if we could just use a TranscriptionFrame to send our transcriber LLM's transcription output down the pipelline. But we can't, because TranscriptionFrame is a child class of TextFrame, which in our pipeline will be interpreted by the TTS service as text that should be turned into speech. We could restructure this pipeline, @@ -200,8 +196,7 @@ class LLMDemoTranscriptionFrame(Frame): class InputTranscriptionFrameEmitter(FrameProcessor): - """ - A simple FrameProcessor that aggregates the TextFrame output from the transcriber LLM + """A simple FrameProcessor that aggregates the TextFrame output from the transcriber LLM and then sends the full response down the pipeline as an LLMDemoTranscriptionFrame. """ @@ -222,8 +217,7 @@ class InputTranscriptionFrameEmitter(FrameProcessor): class TranscriptionContextFixup(FrameProcessor): - """ - This FrameProcessor looks for the LLMDemoTranscriptionFrame and swaps out the + """This FrameProcessor looks for the LLMDemoTranscriptionFrame and swaps out the audio part of the most recent user message with the text transcription. Audio is big, using a lot of tokens and network bandwidth. So doing this is diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 3a73c648a..2f0161412 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -1,26 +1,24 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - +import aiohttp from dotenv import load_dotenv from loguru import logger from runner import configure -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService - from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py index f44d6d9d2..a4369c800 100644 --- a/examples/foundational/26a-gemini-multimodal-live-transcription.py +++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py index 482cc7caf..a11080e92 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py index 50cabc386..3449ed631 100644 --- a/examples/foundational/26c-gemini-multimodal-live-video.py +++ b/examples/foundational/26c-gemini-multimodal-live-video.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 55e6de2b7..b06c0e4ae 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -1,31 +1,29 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure +from simli import SimliConfig + from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame - from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -from runner import configure -from loguru import logger -from dotenv import load_dotenv - -from simli import SimliConfig from pipecat.services.simli import SimliVideoService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py new file mode 100644 index 000000000..4eec1f93e --- /dev/null +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys +from typing import List + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +class TranscriptHandler: + """Simple handler to demonstrate transcript processing. + + Maintains a list of conversation messages and logs them with timestamps. + """ + + def __init__(self): + self.messages: List[TranscriptionMessage] = [] + + async def on_transcript_update( + self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame + ): + """Handle new transcript messages. + + Args: + processor: The TranscriptProcessor that emitted the update + frame: TranscriptionUpdateFrame containing new messages + """ + self.messages.extend(frame.messages) + + # Log the new messages + logger.info("New transcript messages:") + for msg in frame.messages: + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"{timestamp}{msg.role}: {msg.content}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + None, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o", + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript = TranscriptProcessor() + transcript_handler = TranscriptHandler() + + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + transcript.assistant(), # Assistant transcripts + ] + ) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py new file mode 100644 index 000000000..5ee6d60ed --- /dev/null +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -0,0 +1,137 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys +from typing import List + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +class TranscriptHandler: + """Simple handler to demonstrate transcript processing. + + Maintains a list of conversation messages and logs them with timestamps. + """ + + def __init__(self): + self.messages: List[TranscriptionMessage] = [] + + async def on_transcript_update( + self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame + ): + """Handle new transcript messages. + + Args: + processor: The TranscriptProcessor that emitted the update + frame: TranscriptionUpdateFrame containing new messages + """ + self.messages.extend(frame.messages) + + # Log the new messages + logger.info("New transcript messages:") + for msg in frame.messages: + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"{timestamp}{msg.role}: {msg.content}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + None, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20241022" + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", + }, + {"role": "user", "content": "Say hello."}, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript = TranscriptProcessor() + transcript_handler = TranscriptHandler() + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + transcript.assistant(), # Assistant transcripts + ] + ) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py new file mode 100644 index 000000000..37c0fae43 --- /dev/null +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys +from typing import List + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.google import GoogleLLMService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +class TranscriptHandler: + """Simple handler to demonstrate transcript processing. + + Maintains a list of conversation messages and logs them with timestamps. + """ + + def __init__(self): + self.messages: List[TranscriptionMessage] = [] + + async def on_transcript_update( + self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame + ): + """Handle new transcript messages. + + Args: + processor: The TranscriptProcessor that emitted the update + frame: TranscriptionUpdateFrame containing new messages + """ + self.messages.extend(frame.messages) + + # Log the new messages + logger.info("New transcript messages:") + for msg in frame.messages: + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"{timestamp}{msg.role}: {msg.content}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + None, + "Respond bot", + DailyParams( + audio_out_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, + ), + ) + + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = GoogleLLMService( + model="models/gemini-2.0-flash-exp", + # model="gemini-exp-1114", + api_key=os.getenv("GOOGLE_API_KEY"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", + }, + {"role": "user", "content": "Say hello."}, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript = TranscriptProcessor() + transcript_handler = TranscriptHandler() + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + transcript.assistant(), # Assistant transcripts + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/runner.py b/examples/foundational/runner.py index 13c4ff076..55c6a33d3 100644 --- a/examples/foundational/runner.py +++ b/examples/foundational/runner.py @@ -1,13 +1,14 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 54c2013b4..aae6c0987 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -1,25 +1,28 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger from PIL import Image +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, + Frame, ImageRawFrame, + LLMMessagesFrame, OutputImageRawFrame, SpriteFrame, - Frame, - LLMMessagesFrame, TextFrame, UserImageRawFrame, UserImageRequestFrame, @@ -37,12 +40,6 @@ from pipecat.services.moondream import MoondreamService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) @@ -71,8 +68,7 @@ talking_frame = SpriteFrame(images=sprites) class TalkingAnimation(FrameProcessor): - """ - This class starts a talking animation when it receives an first AudioFrame, + """This class starts a talking animation when it receives an first AudioFrame, and then returns to a "quiet" sprite when it sees a TTSStoppedFrame. """ diff --git a/examples/moondream-chatbot/runner.py b/examples/moondream-chatbot/runner.py index 3df3ee81f..8924e0370 100644 --- a/examples/moondream-chatbot/runner.py +++ b/examples/moondream-chatbot/runner.py @@ -1,13 +1,14 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/moondream-chatbot/server.py b/examples/moondream-chatbot/server.py index aa4e82c90..1d2c021be 100644 --- a/examples/moondream-chatbot/server.py +++ b/examples/moondream-chatbot/server.py @@ -1,17 +1,16 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index c33a2495d..a7bc6c925 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -1,32 +1,30 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio -import aiohttp import os import sys import wave +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import OutputAudioRawFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.logger import FrameLogger from pipecat.processors.frame_processor import FrameDirection +from pipecat.processors.logger import FrameLogger from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/patient-intake/runner.py b/examples/patient-intake/runner.py index 3df3ee81f..8924e0370 100644 --- a/examples/patient-intake/runner.py +++ b/examples/patient-intake/runner.py @@ -1,13 +1,14 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/patient-intake/server.py b/examples/patient-intake/server.py index 20894b019..51b8d95eb 100644 --- a/examples/patient-intake/server.py +++ b/examples/patient-intake/server.py @@ -1,17 +1,16 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse diff --git a/examples/simple-chatbot/README.md b/examples/simple-chatbot/README.md index e9b06855d..4e23c9fae 100644 --- a/examples/simple-chatbot/README.md +++ b/examples/simple-chatbot/README.md @@ -24,12 +24,12 @@ This repository demonstrates a simple AI chatbot with real-time audio/video inte 2. **JavaScript** - - Basic implementation using [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/reference/js/introduction) + - Basic implementation using [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/js/introduction) - No framework dependencies - Good for learning the fundamentals 3. **React** - - Basic impelmentation using [Pipecat React SDK](https://docs.pipecat.ai/client/reference/react/introduction) + - Basic impelmentation using [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction) - Demonstrates the basic client principles with Pipecat React ## Quick Start diff --git a/examples/simple-chatbot/examples/javascript/README.md b/examples/simple-chatbot/examples/javascript/README.md index 74525c1c1..fdd97ce27 100644 --- a/examples/simple-chatbot/examples/javascript/README.md +++ b/examples/simple-chatbot/examples/javascript/README.md @@ -1,6 +1,6 @@ # JavaScript Implementation -Basic implementation using the [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/reference/js/introduction). +Basic implementation using the [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/js/introduction). ## Setup diff --git a/examples/simple-chatbot/examples/javascript/src/app.js b/examples/simple-chatbot/examples/javascript/src/app.js index a844453c4..6a32f1fc1 100644 --- a/examples/simple-chatbot/examples/javascript/src/app.js +++ b/examples/simple-chatbot/examples/javascript/src/app.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2024, Daily + * Copyright (c) 2025, Daily * * SPDX-License-Identifier: BSD 2-Clause License */ diff --git a/examples/simple-chatbot/examples/react/README.md b/examples/simple-chatbot/examples/react/README.md index 44775a083..892763d18 100644 --- a/examples/simple-chatbot/examples/react/README.md +++ b/examples/simple-chatbot/examples/react/README.md @@ -1,6 +1,6 @@ # React Implementation -Basic implementation using the [Pipecat React SDK](https://docs.pipecat.ai/client/reference/react/introduction). +Basic implementation using the [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction). ## Setup diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index 991df1cd1..b195d02f1 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -43,7 +43,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import ( RTVIBotTranscriptionProcessor, + RTVIConfig, RTVIMetricsProcessor, + RTVIProcessor, RTVISpeakingProcessor, RTVIUserTranscriptionProcessor, ) @@ -179,9 +181,13 @@ async def main(): # This will send `metrics` messages. rtvi_metrics = RTVIMetricsProcessor() + # Handles RTVI messages from the client + rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + pipeline = Pipeline( [ transport.input(), + rtvi, context_aggregator.user(), llm, rtvi_speaking, @@ -204,6 +210,10 @@ async def main(): ) await task.queue_frame(quiet_frame) + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + await rtvi.set_bot_ready() + @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): await transport.capture_participant_transcription(participant["id"]) diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index a3a68c839..90c5e08d8 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -44,7 +44,9 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import ( RTVIBotTranscriptionProcessor, + RTVIConfig, RTVIMetricsProcessor, + RTVIProcessor, RTVISpeakingProcessor, RTVIUserTranscriptionProcessor, ) @@ -201,9 +203,13 @@ async def main(): # This will send `metrics` messages. rtvi_metrics = RTVIMetricsProcessor() + # Handles RTVI messages from the client + rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + pipeline = Pipeline( [ transport.input(), + rtvi, rtvi_speaking, rtvi_user_transcription, context_aggregator.user(), @@ -227,6 +233,10 @@ async def main(): ) await task.queue_frame(quiet_frame) + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + await rtvi.set_bot_ready() + @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): await transport.capture_participant_transcription(participant["id"]) diff --git a/examples/simple-chatbot/server/runner.py b/examples/simple-chatbot/server/runner.py index 0c03a96bf..ed20d985e 100644 --- a/examples/simple-chatbot/server/runner.py +++ b/examples/simple-chatbot/server/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/simple-chatbot/server/server.py b/examples/simple-chatbot/server/server.py index 7dc6d1f5c..8fb0aa1fc 100644 --- a/examples/simple-chatbot/server/server.py +++ b/examples/simple-chatbot/server/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/storytelling-chatbot/frontend/package-lock.json b/examples/storytelling-chatbot/frontend/package-lock.json index 7b491d0f3..ce4450613 100644 --- a/examples/storytelling-chatbot/frontend/package-lock.json +++ b/examples/storytelling-chatbot/frontend/package-lock.json @@ -16,7 +16,7 @@ "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "framer-motion": "^11.9.0", - "next": "^14.2.14", + "next": "^14.2.15", "react": "^18.3.1", "react-dom": "^18.3.1", "recoil": "^0.7.7", @@ -1912,9 +1912,9 @@ "dev": true }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -4047,9 +4047,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", "funding": [ { "type": "github", diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/src/bot_runner.py index 25a1bca37..4c4e0dcbc 100644 --- a/examples/storytelling-chatbot/src/bot_runner.py +++ b/examples/storytelling-chatbot/src/bot_runner.py @@ -1,34 +1,30 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse -import subprocess import os - +import subprocess +from contextlib import asynccontextmanager from pathlib import Path from typing import Optional -from contextlib import asynccontextmanager - -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles from pipecat.transports.services.helpers.daily_rest import ( DailyRESTHelper, DailyRoomObject, - DailyRoomProperties, DailyRoomParams, + DailyRoomProperties, ) - -from dotenv import load_dotenv - load_dotenv(override=True) # ------------ Fast API Config ------------ # @@ -158,8 +154,7 @@ async def catch_all(path_name: Optional[str] = ""): async def virtualize_bot(room_url: str, token: str): - """ - This is an example of how to virtualize the bot using Fly.io + """This is an example of how to virtualize the bot using Fly.io You can adapt this method to use whichever cloud provider you prefer. """ FLY_API_HOST = os.getenv("FLY_API_HOST", "https://api.machines.dev/v1") diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/src/processors.py index 6aa9ad7ab..096efd577 100644 --- a/examples/storytelling-chatbot/src/processors.py +++ b/examples/storytelling-chatbot/src/processors.py @@ -1,6 +1,8 @@ import re from async_timeout import timeout +from prompts import CUE_ASSISTANT_TURN, CUE_USER_TURN, IMAGE_GEN_PROMPT +from utils.helpers import load_sounds from pipecat.frames.frames import ( Frame, @@ -11,9 +13,6 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.services.daily import DailyTransportMessageFrame -from utils.helpers import load_sounds -from prompts import IMAGE_GEN_PROMPT, CUE_USER_TURN, CUE_ASSISTANT_TURN - sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"]) # -------------- Frame Types ------------- # diff --git a/examples/storytelling-chatbot/src/utils/helpers.py b/examples/storytelling-chatbot/src/utils/helpers.py index 36ba3e609..ac3a38ede 100644 --- a/examples/storytelling-chatbot/src/utils/helpers.py +++ b/examples/storytelling-chatbot/src/utils/helpers.py @@ -1,5 +1,6 @@ import os import wave + from PIL import Image from pipecat.frames.frames import OutputAudioRawFrame, OutputImageRawFrame diff --git a/examples/studypal/env.example b/examples/studypal/env.example index 69245a3d6..6acef284b 100644 --- a/examples/studypal/env.example +++ b/examples/studypal/env.example @@ -1,4 +1,4 @@ -DAILY_SAMPLE_ROOM_URL= # Follow instructions here and put your https://YOURDOMAIN.daily.co/YOURROOM (Instructions: https://docs.pipecat.ai/quickstart#preparing-your-environment) +DAILY_SAMPLE_ROOM_URL= # Follow instructions here and put your https://YOURDOMAIN.daily.co/YOURROOM (Instructions: https://docs.pipecat.ai/getting-started/installation) DAILY_API_KEY= # Create here: https://dashboard.daily.co/developers OPENAI_API_KEY= # Create here: https://platform.openai.com/docs/overview CARTESIA_API_KEY= # Create here: https://play.cartesia.ai/console diff --git a/examples/studypal/runner.py b/examples/studypal/runner.py index 13c4ff076..55c6a33d3 100644 --- a/examples/studypal/runner.py +++ b/examples/studypal/runner.py @@ -1,13 +1,14 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/studypal/studypal.py b/examples/studypal/studypal.py index bccd7ad27..0bed8dedb 100644 --- a/examples/studypal/studypal.py +++ b/examples/studypal/studypal.py @@ -1,12 +1,15 @@ -import aiohttp import asyncio +import io import os import sys -import io -from bs4 import BeautifulSoup -from pypdf import PdfReader +import aiohttp import tiktoken +from bs4 import BeautifulSoup +from dotenv import load_dotenv +from loguru import logger +from pypdf import PdfReader +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -18,12 +21,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) # Run this script directly from your command line. diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index e654c0159..1404b3698 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -1,14 +1,18 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import Frame, LLMMessagesFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -25,12 +29,6 @@ from pipecat.transports.services.daily import ( DailyTransportMessageFrame, ) -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/translation-chatbot/runner.py b/examples/translation-chatbot/runner.py index f19fcf211..8924e0370 100644 --- a/examples/translation-chatbot/runner.py +++ b/examples/translation-chatbot/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/translation-chatbot/server.py b/examples/translation-chatbot/server.py index 9063e28b1..2f8104738 100644 --- a/examples/translation-chatbot/server.py +++ b/examples/translation-chatbot/server.py @@ -1,17 +1,16 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index 5e6d91910..57d542e24 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -1,24 +1,23 @@ import os import sys +from dotenv import load_dotenv +from loguru import logger + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.openai import OpenAILLMService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.transports.network.fastapi_websocket import ( - FastAPIWebsocketTransport, - FastAPIWebsocketParams, -) from pipecat.serializers.twilio import TwilioFrameSerializer - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.network.fastapi_websocket import ( + FastAPIWebsocketParams, + FastAPIWebsocketTransport, +) load_dotenv(override=True) diff --git a/examples/twilio-chatbot/server.py b/examples/twilio-chatbot/server.py index 31e98e25f..c59b6e2ee 100644 --- a/examples/twilio-chatbot/server.py +++ b/examples/twilio-chatbot/server.py @@ -1,13 +1,11 @@ import json import uvicorn - +from bot import run_bot from fastapi import FastAPI, WebSocket from fastapi.middleware.cors import CORSMiddleware from starlette.responses import HTMLResponse -from bot import run_bot - app = FastAPI() app.add_middleware( diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index 3f961de8d..fd792d0df 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,6 +8,9 @@ import asyncio import os import sys +from dotenv import load_dotenv +from loguru import logger + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -22,10 +25,6 @@ from pipecat.transports.network.websocket_server import ( WebsocketServerTransport, ) -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/websocket-server/frames.proto b/examples/websocket-server/frames.proto index 4c58d2a34..484042f18 100644 --- a/examples/websocket-server/frames.proto +++ b/examples/websocket-server/frames.proto @@ -1,5 +1,5 @@ // -// Copyright (c) 2024, Daily +// Copyright (c) 2025, Daily // // SPDX-License-Identifier: BSD 2-Clause License // diff --git a/pyproject.toml b/pyproject.toml index f3f7d6050..1d8b84679 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,14 +21,17 @@ classifiers = [ ] dependencies = [ "aiohttp~=3.11.10", - "loguru~=0.7.2", + "audioop-lts~=0.2.1; python_version>='3.13'", + "loguru~=0.7.3", "Markdown~=3.7", - "numpy~=1.26.4", - "Pillow~=10.4.0", + "numpy~=2.1.3", + "numba~=0.61.0rc1", + "Pillow~=11.0.0", "protobuf~=5.29.1", - "pydantic~=2.8.2", + "pydantic~=2.10.3", "pyloudnorm~=0.1.1", "resampy~=0.4.3", + "tenacity~=9.0.0" ] [project.urls] @@ -42,11 +45,12 @@ aws = [ "boto3~=1.35.27" ] azure = [ "azure-cognitiveservices-speech~=1.41.1", "openai~=1.57.2" ] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.0.13", "websockets~=13.1" ] -daily = [ "daily-python~=0.13.0" ] +cerebras = [ "openai~=1.57.2" ] +daily = [ "daily-python~=0.14.2" ] deepgram = [ "deepgram-sdk~=3.7.7" ] elevenlabs = [ "websockets~=13.1" ] -examples = [ "python-dotenv~=1.0.1", "flask~=3.0.3", "flask_cors~=4.0.1" ] fal = [ "fal-client~=0.4.1" ] +fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.21.1" ] grok = [ "openai~=1.57.2" ] @@ -54,23 +58,24 @@ groq = [ "openai~=1.57.2" ] gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.57.2" ] krisp = [ "pipecat-ai-krisp~=0.3.0" ] -langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] -livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1", "tenacity~=8.5.0" ] +koala = [ "pvkoala~=2.0.2" ] +langchain = [ "langchain~=0.3.12", "langchain-community~=0.3.12", "langchain-openai~=0.2.12" ] +livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1" ] lmnt = [ "lmnt~=1.1.4" ] local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.8", "transformers~=4.44.0" ] nim = [ "openai~=1.57.2" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "openai~=1.57.2", "websockets~=13.1", "python-deepcompare~=1.0.1" ] -openpipe = [ "openpipe~=4.38.0" ] -playht = [ "pyht~=0.1.8", "websockets~=13.1" ] +openpipe = [ "openpipe~=4.40.0" ] +playht = [ "pyht~=0.1.9", "websockets~=13.1" ] riva = [ "nvidia-riva-client~=2.17.0" ] silero = [ "onnxruntime~=1.20.1" ] +simli = [ "simli-ai~=0.1.7"] soundfile = [ "soundfile~=0.12.1" ] together = [ "openai~=1.57.2" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.0" ] whisper = [ "faster-whisper~=1.1.0" ] -simli = [ "simli-ai~=0.1.7"] [tool.setuptools.packages.find] # All the following settings are optional: @@ -87,9 +92,11 @@ fallback_version = "0.0.0-dev" exclude = ["*_pb2.py"] line-length = 100 +[tool.ruff.lint] select = [ "D", # Docstring rules + "I", # Import rules ] -[tool.ruff.pydocstyle] -convention = "google" \ No newline at end of file +[tool.ruff.lint.pydocstyle] +convention = "google" diff --git a/src/pipecat/audio/filters/base_audio_filter.py b/src/pipecat/audio/filters/base_audio_filter.py index e635bb1ba..58ecca33c 100644 --- a/src/pipecat/audio/filters/base_audio_filter.py +++ b/src/pipecat/audio/filters/base_audio_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/filters/koala_filter.py b/src/pipecat/audio/filters/koala_filter.py new file mode 100644 index 000000000..ebd0b1ed4 --- /dev/null +++ b/src/pipecat/audio/filters/koala_filter.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Sequence + +import numpy as np +from loguru import logger + +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame + +try: + import pvkoala +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the Koala filter, you need to `pip install pipecat-ai[koala]`.") + raise Exception(f"Missing module: {e}") + + +class KoalaFilter(BaseAudioFilter): + """This is an audio filter that uses Koala Noise Suppression (from + PicoVoice). + + """ + + def __init__(self, *, access_key: str) -> None: + self._access_key = access_key + + self._filtering = True + self._sample_rate = 0 + self._koala = pvkoala.create(access_key=f"{self._access_key}") + self._koala_ready = True + self._audio_buffer = bytearray() + + async def start(self, sample_rate: int): + self._sample_rate = sample_rate + if self._sample_rate != self._koala.sample_rate: + logger.warning( + f"Koala filter needs sample rate {self._koala.sample_rate} (got {self._sample_rate})" + ) + self._koala_ready = False + + async def stop(self): + self._koala.reset() + + async def process_frame(self, frame: FilterControlFrame): + if isinstance(frame, FilterEnableFrame): + self._filtering = frame.enable + + async def filter(self, audio: bytes) -> bytes: + if not self._koala_ready or not self._filtering: + return audio + + self._audio_buffer.extend(audio) + + filtered_data: Sequence[int] = [] + + num_frames = len(self._audio_buffer) // 2 + while num_frames >= self._koala.frame_length: + # Grab the number of frames required by Koala. + num_bytes = self._koala.frame_length * 2 + audio = bytes(self._audio_buffer[:num_bytes]) + # Process audio + data = np.frombuffer(audio, dtype=np.int16).tolist() + filtered_data += self._koala.process(data) + # Adjust audio buffer and check again + self._audio_buffer = self._audio_buffer[num_bytes:] + num_frames = len(self._audio_buffer) // 2 + + filtered = np.array(filtered_data, dtype=np.int16).tobytes() + + return filtered diff --git a/src/pipecat/audio/filters/krisp_filter.py b/src/pipecat/audio/filters/krisp_filter.py index 0055c672b..7d691f069 100644 --- a/src/pipecat/audio/filters/krisp_filter.py +++ b/src/pipecat/audio/filters/krisp_filter.py @@ -1,14 +1,15 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import numpy as np import os -from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +import numpy as np from loguru import logger + +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame try: @@ -23,8 +24,7 @@ class KrispFilter(BaseAudioFilter): def __init__( self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None ) -> None: - """ - Initializes the KrispAudioProcessor with customizable audio processing settings. + """Initializes the KrispAudioProcessor with customizable audio processing settings. :param sample_type: The type of audio sample, default is 'PCM_16'. :param channels: Number of audio channels, default is 1. diff --git a/src/pipecat/audio/filters/noisereduce_filter.py b/src/pipecat/audio/filters/noisereduce_filter.py index 4f0449452..c97a69c6f 100644 --- a/src/pipecat/audio/filters/noisereduce_filter.py +++ b/src/pipecat/audio/filters/noisereduce_filter.py @@ -1,15 +1,13 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import numpy as np - -from pipecat.audio.filters.base_audio_filter import BaseAudioFilter - from loguru import logger +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame try: diff --git a/src/pipecat/audio/mixers/base_audio_mixer.py b/src/pipecat/audio/mixers/base_audio_mixer.py index 0ba212d85..804948ed4 100644 --- a/src/pipecat/audio/mixers/base_audio_mixer.py +++ b/src/pipecat/audio/mixers/base_audio_mixer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/mixers/soundfile_mixer.py b/src/pipecat/audio/mixers/soundfile_mixer.py index 659ab0507..c49ef1184 100644 --- a/src/pipecat/audio/mixers/soundfile_mixer.py +++ b/src/pipecat/audio/mixers/soundfile_mixer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py index 057942e04..80785237f 100644 --- a/src/pipecat/audio/utils.py +++ b/src/pipecat/audio/utils.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import audioop + import numpy as np import pyloudnorm as pyln import resampy diff --git a/src/pipecat/audio/vad/silero.py b/src/pipecat/audio/vad/silero.py index 1da0fb12d..28df15eba 100644 --- a/src/pipecat/audio/vad/silero.py +++ b/src/pipecat/audio/vad/silero.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -7,11 +7,10 @@ import time import numpy as np +from loguru import logger from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams -from loguru import logger - # How often should we reset internal model state _MODEL_RESET_STATES_TIME = 5.0 diff --git a/src/pipecat/audio/vad/vad_analyzer.py b/src/pipecat/audio/vad/vad_analyzer.py index c291db7f3..6087a38b5 100644 --- a/src/pipecat/audio/vad/vad_analyzer.py +++ b/src/pipecat/audio/vad/vad_analyzer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/clocks/base_clock.py b/src/pipecat/clocks/base_clock.py index 79e17d5ba..e9ed3be80 100644 --- a/src/pipecat/clocks/base_clock.py +++ b/src/pipecat/clocks/base_clock.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/clocks/system_clock.py b/src/pipecat/clocks/system_clock.py index d919b6acd..4c434fc70 100644 --- a/src/pipecat/clocks/system_clock.py +++ b/src/pipecat/clocks/system_clock.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/frames/frames.proto b/src/pipecat/frames/frames.proto index 4c58d2a34..484042f18 100644 --- a/src/pipecat/frames/frames.proto +++ b/src/pipecat/frames/frames.proto @@ -1,5 +1,5 @@ // -// Copyright (c) 2024, Daily +// Copyright (c) 2025, Daily // // SPDX-License-Identifier: BSD 2-Clause License // diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index d3792f537..ac0075a59 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1,11 +1,11 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # from dataclasses import dataclass, field -from typing import Any, List, Mapping, Optional, Tuple +from typing import Any, List, Literal, Mapping, Optional, Tuple from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.clocks.base_clock import BaseClock @@ -195,7 +195,8 @@ class TranscriptionFrame(TextFrame): @dataclass class InterimTranscriptionFrame(TextFrame): """A text frame with interim transcription-specific data. Will be placed in - the transport's receive queue when a participant speaks.""" + the transport's receive queue when a participant speaks. + """ text: str user_id: str @@ -206,6 +207,69 @@ class InterimTranscriptionFrame(TextFrame): return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})" +@dataclass +class OpenAILLMContextAssistantTimestampFrame(DataFrame): + """Timestamp information for assistant message in LLM context.""" + + timestamp: str + + +@dataclass +class TranscriptionMessage: + """A message in a conversation transcript containing the role and content. + + Messages are in standard format with roles normalized to user/assistant. + """ + + role: Literal["user", "assistant"] + content: str + timestamp: str | None = None + + +@dataclass +class TranscriptionUpdateFrame(DataFrame): + """A frame containing new messages added to the conversation transcript. + + This frame is emitted when new messages are added to the conversation history, + containing only the newly added messages rather than the full transcript. + Messages have normalized roles (user/assistant) regardless of the LLM service used. + Messages are always in the OpenAI standard message format, which supports both: + + Simple format: + [ + { + "role": "user", + "content": "Hi, how are you?" + }, + { + "role": "assistant", + "content": "Great! And you?" + } + ] + + Content list format: + [ + { + "role": "user", + "content": [{"type": "text", "text": "Hi, how are you?"}] + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "Great! And you?"}] + } + ] + + OpenAI supports both formats. Anthropic and Google messages are converted to the + content list format. + """ + + messages: List[TranscriptionMessage] + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, messages: {len(self.messages)})" + + @dataclass class LLMMessagesFrame(DataFrame): """A frame containing a list of LLM messages. Used to signal that an LLM @@ -546,7 +610,8 @@ class EndFrame(ControlFrame): @dataclass class LLMFullResponseStartFrame(ControlFrame): """Used to indicate the beginning of an LLM response. Following by one or - more TextFrame and a final LLMFullResponseEndFrame.""" + more TextFrame and a final LLMFullResponseEndFrame. + """ pass diff --git a/src/pipecat/metrics/metrics.py b/src/pipecat/metrics/metrics.py index 053708998..c40f68590 100644 --- a/src/pipecat/metrics/metrics.py +++ b/src/pipecat/metrics/metrics.py @@ -1,4 +1,5 @@ from typing import Optional + from pydantic import BaseModel diff --git a/src/pipecat/pipeline/base_pipeline.py b/src/pipecat/pipeline/base_pipeline.py index 393914684..7debb3615 100644 --- a/src/pipecat/pipeline/base_pipeline.py +++ b/src/pipecat/pipeline/base_pipeline.py @@ -1,11 +1,10 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # from abc import abstractmethod - from typing import List from pipecat.processors.frame_processor import FrameProcessor diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 40bfea90d..b277dc7dd 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -1,20 +1,26 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - from itertools import chain -from typing import Awaitable, Callable, List +from typing import Awaitable, Callable, Dict, List +from loguru import logger + +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + StartFrame, + StartInterruptionFrame, + SystemFrame, +) from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame - -from loguru import logger class Source(FrameProcessor): @@ -73,11 +79,10 @@ class ParallelPipeline(BasePipeline): self._sources = [] self._sinks = [] self._seen_ids = set() + self._endframe_counter: Dict[int, int] = {} self._up_queue = asyncio.Queue() self._down_queue = asyncio.Queue() - self._up_task: asyncio.Task | None = None - self._down_task: asyncio.Task | None = None self._pipelines = [] @@ -112,18 +117,19 @@ class ParallelPipeline(BasePipeline): # async def cleanup(self): + await asyncio.gather(*[s.cleanup() for s in self._sources]) await asyncio.gather(*[p.cleanup() for p in self._pipelines]) - - async def _start_tasks(self): - loop = self.get_event_loop() - self._up_task = loop.create_task(self._process_up_queue()) - self._down_task = loop.create_task(self._process_down_queue()) + await asyncio.gather(*[s.cleanup() for s in self._sinks]) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, StartFrame): - await self._start_tasks() + await self._start() + elif isinstance(frame, EndFrame): + self._endframe_counter[frame.id] = len(self._pipelines) + elif isinstance(frame, CancelFrame): + await self._cancel() if direction == FrameDirection.UPSTREAM: # If we get an upstream frame we process it in each sink. @@ -132,16 +138,45 @@ class ParallelPipeline(BasePipeline): # If we get a downstream frame we process it in each source. await asyncio.gather(*[s.queue_frame(frame, direction) for s in self._sources]) - # If we get an EndFrame we stop our queue processing tasks and wait on - # all the pipelines to finish. - if isinstance(frame, (CancelFrame, EndFrame)): - # Use None to indicate when queues should be done processing. - await self._up_queue.put(None) - await self._down_queue.put(None) - if self._up_task: - await self._up_task - if self._down_task: - await self._down_task + # Handle interruptions after everything has been cancelled. + if isinstance(frame, StartInterruptionFrame): + await self._handle_interruption() + # Wait for tasks to finish. + elif isinstance(frame, EndFrame): + await self._stop() + + async def _start(self): + await self._create_tasks() + + async def _stop(self): + # The up task doesn't receive an EndFrame, so we just cancel it. + self._up_task.cancel() + await self._up_task + # The down tasks waits for the last EndFrame send by the internal + # pipelines. + await self._down_task + + async def _cancel(self): + self._up_task.cancel() + await self._up_task + self._down_task.cancel() + await self._down_task + + async def _create_tasks(self): + loop = self.get_event_loop() + self._up_task = loop.create_task(self._process_up_queue()) + self._down_task = loop.create_task(self._process_down_queue()) + + async def _drain_queues(self): + while not self._up_queue.empty: + await self._up_queue.get() + while not self._down_queue.empty: + await self._down_queue.get() + + async def _handle_interruption(self): + await self._cancel() + await self._drain_queues() + await self._create_tasks() async def _parallel_push_frame(self, frame: Frame, direction: FrameDirection): if frame.id not in self._seen_ids: @@ -149,19 +184,33 @@ class ParallelPipeline(BasePipeline): await self.push_frame(frame, direction) async def _process_up_queue(self): - running = True - while running: - frame = await self._up_queue.get() - if frame: + while True: + try: + frame = await self._up_queue.get() await self._parallel_push_frame(frame, FrameDirection.UPSTREAM) - running = frame is not None - self._up_queue.task_done() + self._up_queue.task_done() + except asyncio.CancelledError: + break async def _process_down_queue(self): running = True while running: - frame = await self._down_queue.get() - if frame: - await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM) - running = frame is not None - self._down_queue.task_done() + try: + frame = await self._down_queue.get() + + endframe_counter = self._endframe_counter.get(frame.id, 0) + + # If we have a counter, decrement it. + if endframe_counter > 0: + self._endframe_counter[frame.id] -= 1 + endframe_counter = self._endframe_counter[frame.id] + + # If we don't have a counter or we reached 0, push the frame. + if endframe_counter == 0: + await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM) + + running = not (endframe_counter == 0 and isinstance(frame, EndFrame)) + + self._down_queue.task_done() + except asyncio.CancelledError: + break diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py index 703f911fe..0aefd0bdf 100644 --- a/src/pipecat/pipeline/pipeline.py +++ b/src/pipecat/pipeline/pipeline.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 57b818487..77d093c6a 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -7,11 +7,11 @@ import asyncio import signal +from loguru import logger + from pipecat.pipeline.task import PipelineTask from pipecat.utils.utils import obj_count, obj_id -from loguru import logger - class PipelineRunner: def __init__(self, *, name: str | None = None, handle_sigint: bool = True): diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py index 20f4275e4..2ba1e4695 100644 --- a/src/pipecat/pipeline/sync_parallel_pipeline.py +++ b/src/pipecat/pipeline/sync_parallel_pipeline.py @@ -1,22 +1,21 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - from dataclasses import dataclass from itertools import chain from typing import List +from loguru import logger + from pipecat.frames.frames import ControlFrame, EndFrame, Frame, SystemFrame from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - @dataclass class SyncFrame(ControlFrame): diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index f09013a58..612ffb06a 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -1,13 +1,13 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - from typing import AsyncIterable, Iterable +from loguru import logger from pydantic import BaseModel from pipecat.clocks.base_clock import BaseClock @@ -23,13 +23,11 @@ from pipecat.frames.frames import ( StartFrame, StopTaskFrame, ) -from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData +from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.utils.utils import obj_count, obj_id -from loguru import logger - class PipelineParams(BaseModel): allow_interruptions: bool = False diff --git a/src/pipecat/pipeline/to_be_updated/merge_pipeline.py b/src/pipecat/pipeline/to_be_updated/merge_pipeline.py index 6142a55ea..27a52894b 100644 --- a/src/pipecat/pipeline/to_be_updated/merge_pipeline.py +++ b/src/pipecat/pipeline/to_be_updated/merge_pipeline.py @@ -1,4 +1,5 @@ from typing import List + from pipecat.frames.frames import EndFrame, EndPipeFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index c39a35c82..975c92e9b 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -1,16 +1,16 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # from typing import List, Tuple +from loguru import logger + from pipecat.frames.frames import Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - class GatedAggregator(FrameProcessor): """Accumulate frames, with custom functions to start and stop accumulation. diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index 71a540dd4..7acda1015 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 479746471..349f7290c 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 6e7474c17..ec69407a5 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -113,10 +113,38 @@ class OpenAILLMContext: return json.dumps(msgs) def from_standard_message(self, message): + """Convert from OpenAI message format to OpenAI message format (passthrough). + + OpenAI's format allows both simple string content and structured content: + - Simple: {"role": "user", "content": "Hello"} + - Structured: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + + Since OpenAI is our standard format, this is a passthrough function. + + Args: + message (dict): Message in OpenAI format + + Returns: + dict: Same message, unchanged + """ return message - # convert a message in this LLM's format to one or more messages in OpenAI format def to_standard_messages(self, obj) -> list: + """Convert from OpenAI message format to OpenAI message format (passthrough). + + OpenAI's format is our standard format throughout Pipecat. This function + returns a list containing the original message to maintain consistency with + other LLM services that may need to return multiple messages. + + Args: + obj (dict): Message in OpenAI format with either: + - Simple content: {"role": "user", "content": "Hello"} + - List content: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + + Returns: + list: List containing the original messages, preserving whether + the content was in simple string or structured list format + """ return [obj] def get_messages_for_initializing_history(self): diff --git a/src/pipecat/processors/aggregators/sentence.py b/src/pipecat/processors/aggregators/sentence.py index d0c593a83..3a97e4f66 100644 --- a/src/pipecat/processors/aggregators/sentence.py +++ b/src/pipecat/processors/aggregators/sentence.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index 903019059..2eb44f070 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -1,10 +1,9 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.frames.frames import ( Frame, InterimTranscriptionFrame, @@ -14,6 +13,7 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class ResponseAggregator(FrameProcessor): diff --git a/src/pipecat/processors/aggregators/vision_image_frame.py b/src/pipecat/processors/aggregators/vision_image_frame.py index d07337f06..cea51afba 100644 --- a/src/pipecat/processors/aggregators/vision_image_frame.py +++ b/src/pipecat/processors/aggregators/vision_image_frame.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py index 4f9bc85d0..aad259dae 100644 --- a/src/pipecat/processors/async_generator.py +++ b/src/pipecat/processors/async_generator.py @@ -1,11 +1,10 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - from typing import Any, AsyncGenerator from pipecat.frames.frames import ( @@ -13,7 +12,7 @@ from pipecat.frames.frames import ( EndFrame, Frame, ) -from pipecat.processors.frame_processor import FrameProcessor, FrameDirection +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.serializers.base_serializer import FrameSerializer diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 488a251f0..5afc9ee43 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/audio/vad/silero.py b/src/pipecat/processors/audio/vad/silero.py index 4aa32a163..62ebe267c 100644 --- a/src/pipecat/processors/audio/vad/silero.py +++ b/src/pipecat/processors/audio/vad/silero.py @@ -1,9 +1,11 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # +from loguru import logger + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams, VADState from pipecat.frames.frames import ( @@ -16,8 +18,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - class SileroVAD(FrameProcessor): def __init__( diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py index 11f2e601a..7c691ad11 100644 --- a/src/pipecat/processors/filters/frame_filter.py +++ b/src/pipecat/processors/filters/frame_filter.py @@ -1,12 +1,12 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # from typing import Tuple, Type -from pipecat.frames.frames import ControlFrame, Frame, SystemFrame +from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -23,7 +23,7 @@ class FrameFilter(FrameProcessor): if isinstance(frame, self._types): return True - return isinstance(frame, ControlFrame) or isinstance(frame, SystemFrame) + return isinstance(frame, (EndFrame, SystemFrame)) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index e38cea3e0..860a2c78c 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -1,12 +1,12 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # from typing import Awaitable, Callable -from pipecat.frames.frames import Frame, SystemFrame +from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -24,9 +24,10 @@ class FunctionFilter(FrameProcessor): # Frame processor # - # Ignore system frames and frames that are not following the direction of this gate + # Ignore system frames, end frames and frames that are not following the + # direction of this gate def _should_passthrough_frame(self, frame, direction): - return isinstance(frame, SystemFrame) or direction != self._direction + return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/src/pipecat/processors/filters/identity_filter.py b/src/pipecat/processors/filters/identity_filter.py index d6f896b73..78cba5186 100644 --- a/src/pipecat/processors/filters/identity_filter.py +++ b/src/pipecat/processors/filters/identity_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/null_filter.py b/src/pipecat/processors/filters/null_filter.py index 7e9ca6725..aae59a3c2 100644 --- a/src/pipecat/processors/filters/null_filter.py +++ b/src/pipecat/processors/filters/null_filter.py @@ -1,10 +1,11 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.processors.frame_processor import FrameProcessor +from pipecat.frames.frames import EndFrame, Frame, SystemFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class NullFilter(FrameProcessor): @@ -12,3 +13,13 @@ class NullFilter(FrameProcessor): def __init__(self, **kwargs): super().__init__(**kwargs) + + # + # Frame processor + # + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, (SystemFrame, EndFrame)): + await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index 914198e5a..bc709229a 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index f1a7afbef..a9402c0c1 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -1,23 +1,21 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import re import time - from enum import Enum +from loguru import logger + from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - class WakeCheckFilter(FrameProcessor): - """ - This filter looks for wake phrases in the transcription frames and only passes through frames + """This filter looks for wake phrases in the transcription frames and only passes through frames after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief period of continued conversation after a wake phrase has been detected. """ diff --git a/src/pipecat/processors/filters/wake_notifier_filter.py b/src/pipecat/processors/filters/wake_notifier_filter.py index a7f074ccb..e0e0cce3a 100644 --- a/src/pipecat/processors/filters/wake_notifier_filter.py +++ b/src/pipecat/processors/filters/wake_notifier_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 52066b4f4..ad15de62e 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -268,6 +268,7 @@ class FrameProcessor: raise def __create_input_task(self): + self.__should_block_frames = False self.__input_queue = asyncio.Queue() self.__input_frame_task = self.get_event_loop().create_task( self.__input_frame_task_handler() diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index c0b657244..d1a691b2f 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -1,11 +1,13 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # from typing import Union +from loguru import logger + from pipecat.frames.frames import ( Frame, LLMFullResponseEndFrame, @@ -15,8 +17,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - try: from langchain_core.messages import AIMessageChunk from langchain_core.runnables import Runnable diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 471bdbb88..92ff01df4 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 649a2c529..9de5ff08e 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -1,11 +1,12 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio +from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( @@ -19,8 +20,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - try: import gi diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index e674b6b84..b8b447f94 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -1,11 +1,10 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - from typing import Awaitable, Callable, List from pipecat.frames.frames import Frame diff --git a/src/pipecat/processors/logger.py b/src/pipecat/processors/logger.py index a26c67014..d4f2615a2 100644 --- a/src/pipecat/processors/logger.py +++ b/src/pipecat/processors/logger.py @@ -1,14 +1,16 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.frames.frames import BotSpeakingFrame, Frame, AudioRawFrame, TransportMessageFrame -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger from typing import Optional +from loguru import logger + +from pipecat.frames.frames import AudioRawFrame, BotSpeakingFrame, Frame, TransportMessageFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + logger = logger.opt(ansi=True) diff --git a/src/pipecat/processors/metrics/frame_processor_metrics.py b/src/pipecat/processors/metrics/frame_processor_metrics.py index a22639239..8d1ba4191 100644 --- a/src/pipecat/processors/metrics/frame_processor_metrics.py +++ b/src/pipecat/processors/metrics/frame_processor_metrics.py @@ -1,5 +1,13 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import time +from loguru import logger + from pipecat.frames.frames import MetricsFrame from pipecat.metrics.metrics import ( LLMTokenUsage, @@ -10,8 +18,6 @@ from pipecat.metrics.metrics import ( TTSUsageMetricsData, ) -from loguru import logger - class FrameProcessorMetrics: def __init__(self): diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index e37dd9d44..5a092dc98 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -1,4 +1,11 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import time + from loguru import logger try: @@ -29,8 +36,9 @@ class SentryMetrics(FrameProcessorMetrics): description=f"TTFB for {self._processor_name()}", start_timestamp=self._start_ttfb_time, ) - logger.debug(f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: { - self._ttfb_metrics_span.description} started.") + logger.debug( + f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: {self._ttfb_metrics_span.description} started." + ) self._should_report_ttfb = not report_only_initial_ttfb async def stop_ttfb_metrics(self): @@ -46,8 +54,9 @@ class SentryMetrics(FrameProcessorMetrics): description=f"Processing for {self._processor_name()}", start_timestamp=self._start_processing_time, ) - logger.debug(f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: { - self._processing_metrics_span.description} started.") + logger.debug( + f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: {self._processing_metrics_span.description} started." + ) async def stop_processing_metrics(self): stop_time = time.time() diff --git a/src/pipecat/processors/text_transformer.py b/src/pipecat/processors/text_transformer.py index 79e9b885e..5e84551ed 100644 --- a/src/pipecat/processors/text_transformer.py +++ b/src/pipecat/processors/text_transformer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py new file mode 100644 index 000000000..884d4988f --- /dev/null +++ b/src/pipecat/processors/transcript_processor.py @@ -0,0 +1,252 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import List + +from pipecat.frames.frames import ( + Frame, + OpenAILLMContextAssistantTimestampFrame, + TranscriptionFrame, + TranscriptionMessage, + TranscriptionUpdateFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + + +class BaseTranscriptProcessor(FrameProcessor): + """Base class for processing conversation transcripts. + + Provides common functionality for handling transcript messages and updates. + """ + + def __init__(self, **kwargs): + """Initialize processor with empty message store.""" + super().__init__(**kwargs) + self._processed_messages: List[TranscriptionMessage] = [] + self._register_event_handler("on_transcript_update") + + async def _emit_update(self, messages: List[TranscriptionMessage]): + """Emit transcript updates for new messages. + + Args: + messages: New messages to emit in update + """ + if messages: + self._processed_messages.extend(messages) + update_frame = TranscriptionUpdateFrame(messages=messages) + await self._call_event_handler("on_transcript_update", update_frame) + await self.push_frame(update_frame) + + +class UserTranscriptProcessor(BaseTranscriptProcessor): + """Processes user transcription frames into timestamped conversation messages.""" + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process TranscriptionFrames into user conversation messages. + + Args: + frame: Input frame to process + direction: Frame processing direction + """ + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + message = TranscriptionMessage( + role="user", content=frame.text, timestamp=frame.timestamp + ) + await self._emit_update([message]) + + await self.push_frame(frame, direction) + + +class AssistantTranscriptProcessor(BaseTranscriptProcessor): + """Processes assistant LLM context frames into timestamped conversation messages.""" + + def __init__(self, **kwargs): + """Initialize processor with empty message stores.""" + super().__init__(**kwargs) + self._pending_assistant_messages: List[TranscriptionMessage] = [] + + def _extract_messages(self, messages: List[dict]) -> List[TranscriptionMessage]: + """Extract assistant messages from the OpenAI standard message format. + + Args: + messages: List of messages in OpenAI format, which can be either: + - Simple format: {"role": "user", "content": "Hello"} + - Content list: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + + Returns: + List[TranscriptionMessage]: Normalized conversation messages + """ + result = [] + for msg in messages: + if msg["role"] != "assistant": + continue + + content = msg.get("content") + if isinstance(content, str): + if content: + result.append(TranscriptionMessage(role="assistant", content=content)) + elif isinstance(content, list): + text_parts = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text_parts.append(part["text"]) + + if text_parts: + result.append( + TranscriptionMessage(role="assistant", content=" ".join(text_parts)) + ) + + return result + + def _find_new_messages(self, current: List[TranscriptionMessage]) -> List[TranscriptionMessage]: + """Find unprocessed messages from current list. + + Args: + current: List of current messages + + Returns: + List[TranscriptionMessage]: New messages not yet processed + """ + if not self._processed_messages: + return current + + processed_len = len(self._processed_messages) + if len(current) <= processed_len: + return [] + + return current[processed_len:] + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process frames into assistant conversation messages. + + Args: + frame: Input frame to process + direction: Frame processing direction + """ + await super().process_frame(frame, direction) + + if isinstance(frame, OpenAILLMContextFrame): + standard_messages = [] + for msg in frame.context.messages: + converted = frame.context.to_standard_messages(msg) + standard_messages.extend(converted) + + current_messages = self._extract_messages(standard_messages) + new_messages = self._find_new_messages(current_messages) + self._pending_assistant_messages.extend(new_messages) + + elif isinstance(frame, OpenAILLMContextAssistantTimestampFrame): + if self._pending_assistant_messages: + for msg in self._pending_assistant_messages: + msg.timestamp = frame.timestamp + await self._emit_update(self._pending_assistant_messages) + self._pending_assistant_messages = [] + + await self.push_frame(frame, direction) + + +class TranscriptProcessor: + """Factory for creating and managing transcript processors. + + Provides unified access to user and assistant transcript processors + with shared event handling. + + Example: + ```python + transcript = TranscriptProcessor() + + pipeline = Pipeline( + [ + transport.input(), + stt, + transcript.user(), # User transcripts + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + transcript.assistant(), # Assistant transcripts + ] + ) + + @transcript.event_handler("on_transcript_update") + async def handle_update(processor, frame): + print(f"New messages: {frame.messages}") + ``` + """ + + def __init__(self): + """Initialize factory.""" + self._user_processor = None + self._assistant_processor = None + self._event_handlers = {} + + def user(self, **kwargs) -> UserTranscriptProcessor: + """Get the user transcript processor. + + Args: + **kwargs: Arguments specific to UserTranscriptProcessor + """ + if self._user_processor is None: + self._user_processor = UserTranscriptProcessor(**kwargs) + # Apply any registered event handlers + for event_name, handler in self._event_handlers.items(): + + @self._user_processor.event_handler(event_name) + async def user_handler(processor, frame): + return await handler(processor, frame) + + return self._user_processor + + def assistant(self, **kwargs) -> AssistantTranscriptProcessor: + """Get the assistant transcript processor. + + Args: + **kwargs: Arguments specific to AssistantTranscriptProcessor + """ + if self._assistant_processor is None: + self._assistant_processor = AssistantTranscriptProcessor(**kwargs) + # Apply any registered event handlers + for event_name, handler in self._event_handlers.items(): + + @self._assistant_processor.event_handler(event_name) + async def assistant_handler(processor, frame): + return await handler(processor, frame) + + return self._assistant_processor + + def event_handler(self, event_name: str): + """Register event handler for both processors. + + Args: + event_name: Name of event to handle + + Returns: + Decorator function that registers handler with both processors + """ + + def decorator(handler): + self._event_handlers[event_name] = handler + + # Apply handler to existing processors if they exist + if self._user_processor: + + @self._user_processor.event_handler(event_name) + async def user_handler(processor, frame): + return await handler(processor, frame) + + if self._assistant_processor: + + @self._assistant_processor.event_handler(event_name) + async def assistant_handler(processor, frame): + return await handler(processor, frame) + + return handler + + return decorator diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 160c49908..e5bbbdd1c 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/serializers/base_serializer.py b/src/pipecat/serializers/base_serializer.py index 00f2f2a68..c421ca46c 100644 --- a/src/pipecat/serializers/base_serializer.py +++ b/src/pipecat/serializers/base_serializer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py index a14483b15..50f341e5e 100644 --- a/src/pipecat/serializers/livekit.py +++ b/src/pipecat/serializers/livekit.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -7,11 +7,11 @@ import ctypes import pickle +from loguru import logger + from pipecat.frames.frames import Frame, InputAudioRawFrame, OutputAudioRawFrame from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType -from loguru import logger - try: from livekit.rtc import AudioFrame except ModuleNotFoundError as e: diff --git a/src/pipecat/serializers/protobuf.py b/src/pipecat/serializers/protobuf.py index 1d2e6c3f2..c2179afa2 100644 --- a/src/pipecat/serializers/protobuf.py +++ b/src/pipecat/serializers/protobuf.py @@ -1,13 +1,14 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import dataclasses -import pipecat.frames.protobufs.frames_pb2 as frame_protos +from loguru import logger +import pipecat.frames.protobufs.frames_pb2 as frame_protos from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -17,8 +18,6 @@ from pipecat.frames.frames import ( ) from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType -from loguru import logger - class ProtobufFrameSerializer(FrameSerializer): SERIALIZABLE_TYPES = { @@ -59,7 +58,9 @@ class ProtobufFrameSerializer(FrameSerializer): return proto_frame.SerializeToString() def deserialize(self, data: str | bytes) -> Frame | None: - """Returns a Frame object from a Frame protobuf. Used to convert frames + """Returns a Frame object from a Frame protobuf. + + Used to convert frames passed over the wire as protobufs to Frame objects used in pipelines and frame processors. @@ -76,7 +77,6 @@ class ProtobufFrameSerializer(FrameSerializer): ... text="Hello there!", participantId="123", timestamp="2021-01-01"))) TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01') """ - proto = frame_protos.Frame.FromString(data) which = proto.WhichOneof("frame") if which not in self.DESERIALIZABLE_FIELDS: diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py index a0d02fa2f..577664dde 100644 --- a/src/pipecat/serializers/twilio.py +++ b/src/pipecat/serializers/twilio.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -9,7 +9,7 @@ import json from pydantic import BaseModel -from pipecat.audio.utils import ulaw_to_pcm, pcm_to_ulaw +from pipecat.audio.utils import pcm_to_ulaw, ulaw_to_pcm from pipecat.frames.frames import AudioRawFrame, Frame, InputAudioRawFrame, StartInterruptionFrame from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index e0f16e220..45b1c4639 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 48102eda7..807333c17 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -11,7 +11,7 @@ import json import re from asyncio import CancelledError from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from loguru import logger from PIL import Image @@ -26,6 +26,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMUpdateSettingsFrame, + OpenAILLMContextAssistantTimestampFrame, StartInterruptionFrame, TextFrame, UserImageRawFrame, @@ -43,6 +44,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import LLMService +from pipecat.utils.time import time_now_iso8601 try: from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven @@ -75,8 +77,7 @@ class AnthropicContextAggregatorPair: class AnthropicLLMService(LLMService): - """ - This class implements inference with Anthropic's AI models. + """This class implements inference with Anthropic's AI models. Can provide a custom client via the `client` kwarg, allowing you to use `AsyncAnthropicBedrock` and `AsyncAnthropicVertex` clients @@ -328,7 +329,7 @@ class AnthropicLLMContext(OpenAILLMContext): tools: list[dict] | None = None, tool_choice: dict | None = None, *, - system: str | NotGiven = NOT_GIVEN, + system: Union[str, NotGiven] = NOT_GIVEN, ): super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) @@ -379,6 +380,26 @@ class AnthropicLLMContext(OpenAILLMContext): # convert a message in Anthropic format into one or more messages in OpenAI format def to_standard_messages(self, obj): + """Convert Anthropic message format to standard structured format. + + Handles text content and function calls for both user and assistant messages. + + Args: + obj: Message in Anthropic format: + { + "role": "user/assistant", + "content": str | [{"type": "text/tool_use/tool_result", ...}] + } + + Returns: + List of messages in standard format: + [ + { + "role": "user/assistant/tool", + "content": [{"type": "text", "text": str}] + } + ] + """ # todo: image format (?) # tool_use role = obj.get("role") @@ -433,6 +454,30 @@ class AnthropicLLMContext(OpenAILLMContext): return messages def from_standard_message(self, message): + """Convert standard format message to Anthropic format. + + Handles conversion of text content, tool calls, and tool results. + Empty text content is converted to "(empty)". + + Args: + message: Message in standard format: + { + "role": "user/assistant/tool", + "content": str | [{"type": "text", ...}], + "tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}] + } + + Returns: + Message in Anthropic format: + { + "role": "user/assistant", + "content": str | [ + {"type": "text", "text": str} | + {"type": "tool_use", "id": str, "name": str, "input": dict} | + {"type": "tool_result", "tool_use_id": str, "content": str} + ] + } + """ # todo: image messages (?) if message["role"] == "tool": return { @@ -748,8 +793,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): if run_llm: await self._user_context_aggregator.push_context_frame() + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + except Exception as e: logger.error(f"Error processing frame: {e}") diff --git a/src/pipecat/services/assemblyai.py b/src/pipecat/services/assemblyai.py index a96589e5a..f17cfa903 100644 --- a/src/pipecat/services/assemblyai.py +++ b/src/pipecat/services/assemblyai.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import asyncio from typing import AsyncGenerator @@ -67,8 +73,7 @@ class AssemblyAISTTService(STTService): await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """ - Process an audio chunk for STT transcription. + """Process an audio chunk for STT transcription. This method streams the audio data to AssemblyAI for real-time transcription. Transcription results are handled asynchronously via callback functions. @@ -83,8 +88,7 @@ class AssemblyAISTTService(STTService): yield None async def _connect(self): - """ - Establish a connection to the AssemblyAI real-time transcription service. + """Establish a connection to the AssemblyAI real-time transcription service. This method sets up the necessary callback functions and initializes the AssemblyAI transcriber. @@ -95,8 +99,7 @@ class AssemblyAISTTService(STTService): logger.info(f"{self}: Connected to AssemblyAI") def on_data(transcript: aai.RealtimeTranscript): - """ - Callback for handling incoming transcription data. + """Callback for handling incoming transcription data. This function runs in a separate thread from the main asyncio event loop. It creates appropriate transcription frames and schedules them to be @@ -121,8 +124,7 @@ class AssemblyAISTTService(STTService): asyncio.run_coroutine_threadsafe(self.push_frame(frame), self._loop) def on_error(error: aai.RealtimeError): - """ - Callback for handling errors from AssemblyAI. + """Callback for handling errors from AssemblyAI. Like on_data, this runs in a separate thread and schedules error handling in the main event loop. diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py index cd09e2ff8..b6a763f16 100644 --- a/src/pipecat/services/aws.py +++ b/src/pipecat/services/aws.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index a95ff7d3c..def2ffccb 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/canonical.py b/src/pipecat/services/canonical.py index 265cc1b1b..d2671b250 100644 --- a/src/pipecat/services/canonical.py +++ b/src/pipecat/services/canonical.py @@ -1,26 +1,25 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import io import os import uuid import wave - from datetime import datetime from typing import Dict, List, Tuple +import aiohttp +from loguru import logger + from pipecat.frames.frames import CancelFrame, EndFrame, Frame from pipecat.processors.audio import audio_buffer_processor from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AIService -from loguru import logger - try: import aiofiles import aiofiles.os @@ -44,7 +43,6 @@ class CanonicalMetricsService(AIService): uploads it to Canonical Voice API for audio processing. Args: - call_id (str): Your unique identifier for the call. This is used to match the call in the Canonical Voice system to the call in your system. assistant (str): Identifier for the AI assistant. This can be whatever you want, it's intended for you convenience so you can distinguish between different assistants and a grouping mechanism for calls. diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 8683fd29a..540d7cfd8 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -12,6 +12,7 @@ from typing import AsyncGenerator, List, Optional, Union from loguru import logger from pydantic import BaseModel +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from pipecat.frames.frames import ( BotStoppedSpeakingFrame, @@ -239,52 +240,64 @@ class CartesiaTTSService(WordTTSService): msg = self._build_msg(text="", continue_transcript=False) await self._websocket.send(msg) + async def _receive_messages(self): + async for message in self._get_websocket(): + msg = json.loads(message) + if not msg or msg["context_id"] != self._context_id: + continue + if msg["type"] == "done": + await self.stop_ttfb_metrics() + # Unset _context_id but not the _context_id_start_timestamp + # because we are likely still playing out audio and need the + # timestamp to set send context frames. + self._context_id = None + await self.add_word_timestamps( + [("TTSStoppedFrame", 0), ("LLMFullResponseEndFrame", 0), ("Reset", 0)] + ) + elif msg["type"] == "timestamps": + await self.add_word_timestamps( + list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"])) + ) + elif msg["type"] == "chunk": + await self.stop_ttfb_metrics() + self.start_word_timestamps() + frame = TTSAudioRawFrame( + audio=base64.b64decode(msg["data"]), + sample_rate=self._settings["output_format"]["sample_rate"], + num_channels=1, + ) + await self.push_frame(frame) + elif msg["type"] == "error": + logger.error(f"{self} error: {msg}") + await self.push_frame(TTSStoppedFrame()) + await self.stop_all_metrics() + await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) + else: + logger.error(f"{self} error, unknown message type: {msg}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + async def _receive_task_handler(self): while True: try: - async for message in self._get_websocket(): - msg = json.loads(message) - if not msg or msg["context_id"] != self._context_id: - continue - if msg["type"] == "done": - await self.stop_ttfb_metrics() - # Unset _context_id but not the _context_id_start_timestamp - # because we are likely still playing out audio and need the - # timestamp to set send context frames. - self._context_id = None - await self.add_word_timestamps( - [("TTSStoppedFrame", 0), ("LLMFullResponseEndFrame", 0), ("Reset", 0)] - ) - elif msg["type"] == "timestamps": - await self.add_word_timestamps( - list( - zip( - msg["word_timestamps"]["words"], msg["word_timestamps"]["start"] - ) - ) - ) - elif msg["type"] == "chunk": - await self.stop_ttfb_metrics() - self.start_word_timestamps() - frame = TTSAudioRawFrame( - audio=base64.b64decode(msg["data"]), - sample_rate=self._settings["output_format"]["sample_rate"], - num_channels=1, - ) - await self.push_frame(frame) - elif msg["type"] == "error": - logger.error(f"{self} error: {msg}") - await self.push_frame(TTSStoppedFrame()) - await self.stop_all_metrics() - await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) - else: - logger.error(f"{self} error, unknown message type: {msg}") + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception: {e}") - await self._disconnect_websocket() - await self._connect_websocket() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/src/pipecat/services/cerebras.py b/src/pipecat/services/cerebras.py new file mode 100644 index 000000000..7867d5fae --- /dev/null +++ b/src/pipecat/services/cerebras.py @@ -0,0 +1,85 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import List + +from loguru import logger + +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.openai import OpenAILLMService + +try: + from openai import ( + AsyncStream, + ) + from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Fireworks, you need to `pip install pipecat-ai[cerebras]`. Also, set `CEREBRAS_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + + +class CerebrasLLMService(OpenAILLMService): + """A service for interacting with Cerebras's API using the OpenAI-compatible interface. + + This service extends OpenAILLMService to connect to Cerebras's API endpoint while + maintaining full compatibility with OpenAI's interface and functionality. + + Args: + api_key (str): The API key for accessing Cerebras's API + base_url (str, optional): The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1" + model (str, optional): The model identifier to use. Defaults to "llama-3.3-70b" + **kwargs: Additional keyword arguments passed to OpenAILLMService + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.cerebras.ai/v1", + model: str = "llama-3.3-70b", + **kwargs, + ): + super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) + + def create_client(self, api_key=None, base_url=None, **kwargs): + """Create OpenAI-compatible client for Cerebras API endpoint.""" + logger.debug(f"Creating Cerebras client with api {base_url}") + return super().create_client(api_key, base_url, **kwargs) + + async def get_chat_completions( + self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] + ) -> AsyncStream[ChatCompletionChunk]: + """Create a streaming chat completion using Cerebras's API. + + Args: + context (OpenAILLMContext): The context object containing tools configuration + and other settings for the chat completion. + messages (List[ChatCompletionMessageParam]): The list of messages comprising + the conversation history and current request. + + Returns: + AsyncStream[ChatCompletionChunk]: A streaming response of chat completion + chunks that can be processed asynchronously. + """ + params = { + "model": self.model_name, + "stream": True, + "messages": messages, + "tools": context.tools, + "tool_choice": context.tool_choice, + "seed": self._settings["seed"], + "temperature": self._settings["temperature"], + "top_p": self._settings["top_p"], + "max_completion_tokens": self._settings["max_completion_tokens"], + } + + params.update(self._settings["extra"]) + + chunks = await self._client.chat.completions.create(**params) + return chunks diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 6578f2873..132656385 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -138,6 +138,13 @@ class DeepgramSTTService(STTService): merged_options = default_options if live_options: merged_options = LiveOptions(**{**default_options.to_dict(), **live_options.to_dict()}) + + # deepgram connection requires language to be a string + if isinstance(merged_options.language, Language) and hasattr( + merged_options.language, "value" + ): + merged_options.language = merged_options.language.value + self._settings = merged_options.to_dict() self._client = DeepgramClient( diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index b829f4945..988eecfab 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -11,11 +11,13 @@ from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, from loguru import logger from pydantic import BaseModel, model_validator +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, EndFrame, + ErrorFrame, Frame, LLMFullResponseEndFrame, StartFrame, @@ -41,6 +43,12 @@ except ModuleNotFoundError as e: ElevenLabsOutputFormat = Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"] +ELEVENLABS_MULTILINGUAL_MODELS = { + "eleven_turbo_v2_5", + "eleven_multilingual_v2", + "eleven_flash_v2_5", +} + def language_to_elevenlabs_language(language: Language) -> str | None: BASE_LANGUAGES = { @@ -133,6 +141,7 @@ class ElevenLabsTTSService(WordTTSService): similarity_boost: Optional[float] = None style: Optional[float] = None use_speaker_boost: Optional[bool] = None + auto_mode: Optional[bool] = True @model_validator(mode="after") def validate_voice_settings(self): @@ -149,7 +158,7 @@ class ElevenLabsTTSService(WordTTSService): *, api_key: str, voice_id: str, - model: str = "eleven_turbo_v2_5", + model: str = "eleven_flash_v2_5", url: str = "wss://api.elevenlabs.io", output_format: ElevenLabsOutputFormat = "pcm_24000", params: InputParams = InputParams(), @@ -191,6 +200,7 @@ class ElevenLabsTTSService(WordTTSService): "similarity_boost": params.similarity_boost, "style": params.style, "use_speaker_boost": params.use_speaker_boost, + "auto_mode": str(params.auto_mode).lower(), } self.set_model_name(model) self.set_voice(voice_id) @@ -310,18 +320,18 @@ class ElevenLabsTTSService(WordTTSService): voice_id = self._voice_id model = self.model_name output_format = self._settings["output_format"] - url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}" + url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}" if self._settings["optimize_streaming_latency"]: url += f"&optimize_streaming_latency={self._settings['optimize_streaming_latency']}" - # Language can only be used with the 'eleven_turbo_v2_5' model + # Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS language = self._settings["language"] - if model == "eleven_turbo_v2_5": + if model in ELEVENLABS_MULTILINGUAL_MODELS: url += f"&language_code={language}" else: logger.warning( - f"Language code [{language}] not applied. Language codes can only be used with the 'eleven_turbo_v2_5' model." + f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}" ) self._websocket = await websockets.connect(url) @@ -352,28 +362,44 @@ class ElevenLabsTTSService(WordTTSService): except Exception as e: logger.error(f"{self} error closing websocket: {e}") + async def _receive_messages(self): + async for message in self._websocket: + msg = json.loads(message) + if msg.get("audio"): + await self.stop_ttfb_metrics() + self.start_word_timestamps() + + audio = base64.b64decode(msg["audio"]) + frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1) + await self.push_frame(frame) + if msg.get("alignment"): + word_times = calculate_word_times(msg["alignment"], self._cumulative_time) + await self.add_word_timestamps(word_times) + self._cumulative_time = word_times[-1][1] + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + async def _receive_task_handler(self): while True: try: - async for message in self._websocket: - msg = json.loads(message) - if msg.get("audio"): - await self.stop_ttfb_metrics() - self.start_word_timestamps() - - audio = base64.b64decode(msg["audio"]) - frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1) - await self.push_frame(frame) - if msg.get("alignment"): - word_times = calculate_word_times(msg["alignment"], self._cumulative_time) - await self.add_word_timestamps(word_times) - self._cumulative_time = word_times[-1][1] + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception: {e}") - await self._disconnect_websocket() - await self._connect_websocket() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def _keepalive_task_handler(self): while True: diff --git a/src/pipecat/services/fal.py b/src/pipecat/services/fal.py index aecdeb709..e949a5713 100644 --- a/src/pipecat/services/fal.py +++ b/src/pipecat/services/fal.py @@ -1,23 +1,21 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import io import os +from typing import AsyncGenerator, Dict, Optional, Union +import aiohttp +from loguru import logger +from PIL import Image from pydantic import BaseModel -from typing import AsyncGenerator, Optional, Union, Dict from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame from pipecat.services.ai_services import ImageGenService -from PIL import Image - -from loguru import logger - try: import fal_client except ModuleNotFoundError as e: diff --git a/src/pipecat/services/fireworks.py b/src/pipecat/services/fireworks.py index 1f7d22b36..0f43bf377 100644 --- a/src/pipecat/services/fireworks.py +++ b/src/pipecat/services/fireworks.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py new file mode 100644 index 000000000..91e19163b --- /dev/null +++ b/src/pipecat/services/fish.py @@ -0,0 +1,245 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import uuid +from typing import AsyncGenerator, Literal, Optional + +from loguru import logger +from pydantic import BaseModel +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential + +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + LLMFullResponseEndFrame, + StartFrame, + StartInterruptionFrame, + TTSAudioRawFrame, + TTSSpeakFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import TTSService +from pipecat.transcriptions.language import Language + +try: + import ormsgpack + import websockets +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Fish Audio, you need to `pip install pipecat-ai[fish]`. Also, set `FISH_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + +# FishAudio supports various output formats +FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] + + +class FishAudioTTSService(TTSService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN + latency: Optional[str] = "normal" # "normal" or "balanced" + prosody_speed: Optional[float] = 1.0 # Speech speed (0.5-2.0) + prosody_volume: Optional[int] = 0 # Volume adjustment in dB + + def __init__( + self, + *, + api_key: str, + model: str, # This is the reference_id + output_format: FishAudioOutputFormat = "pcm", + sample_rate: int = 24000, + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + + self._api_key = api_key + self._base_url = "wss://api.fish.audio/v1/tts/live" + self._websocket = None + self._receive_task = None + self._request_id = None + self._started = False + + self._settings = { + "sample_rate": sample_rate, + "latency": params.latency, + "format": output_format, + "prosody": { + "speed": params.prosody_speed, + "volume": params.prosody_volume, + }, + "reference_id": model, + } + + self.set_model_name(model) + + def can_generate_metrics(self) -> bool: + return True + + async def set_model(self, model: str): + self._settings["reference_id"] = model + await super().set_model(model) + logger.info(f"Switching TTS model to: [{model}]") + + async def start(self, frame: StartFrame): + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._disconnect() + + async def _connect(self): + await self._connect_websocket() + self._receive_task = self.get_event_loop().create_task(self._receive_task_handler()) + + async def _disconnect(self): + await self._disconnect_websocket() + if self._receive_task: + self._receive_task.cancel() + await self._receive_task + self._receive_task = None + + async def _connect_websocket(self): + try: + logger.debug("Connecting to Fish Audio") + headers = {"Authorization": f"Bearer {self._api_key}"} + self._websocket = await websockets.connect(self._base_url, extra_headers=headers) + + # Send initial start message with ormsgpack + start_message = {"event": "start", "request": {"text": "", **self._settings}} + await self._websocket.send(ormsgpack.packb(start_message)) + logger.debug("Sent start message to Fish Audio") + except Exception as e: + logger.error(f"Fish Audio initialization error: {e}") + self._websocket = None + + async def _disconnect_websocket(self): + try: + await self.stop_all_metrics() + if self._websocket: + logger.debug("Disconnecting from Fish Audio") + # Send stop event with ormsgpack + stop_message = {"event": "stop"} + await self._websocket.send(ormsgpack.packb(stop_message)) + await self._websocket.close() + self._websocket = None + self._request_id = None + self._started = False + except Exception as e: + logger.error(f"Error closing websocket: {e}") + + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _receive_messages(self): + async for message in self._get_websocket(): + try: + if isinstance(message, bytes): + msg = ormsgpack.unpackb(message) + if isinstance(msg, dict): + event = msg.get("event") + if event == "audio": + audio_data = msg.get("audio") + # Only process larger chunks to remove msgpack overhead + if audio_data and len(audio_data) > 1024: + frame = TTSAudioRawFrame( + audio_data, self._settings["sample_rate"], 1 + ) + await self.push_frame(frame) + await self.stop_ttfb_metrics() + continue + + except Exception as e: + logger.error(f"Error processing message: {e}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"Fish Audio reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + + async def _receive_task_handler(self): + while True: + try: + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() + except asyncio.CancelledError: + break + except Exception as e: + message = f"Fish Audio error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TTSSpeakFrame): + await self.pause_processing_frames() + elif isinstance(frame, LLMFullResponseEndFrame) and self._request_id: + await self.pause_processing_frames() + elif isinstance(frame, BotStoppedSpeakingFrame): + await self.resume_processing_frames() + + async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + await super()._handle_interruption(frame, direction) + await self.stop_all_metrics() + self._request_id = None + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + logger.debug(f"Generating Fish TTS: [{text}]") + try: + if not self._websocket or self._websocket.closed: + await self._connect() + + if not self._request_id: + await self.start_ttfb_metrics() + await self.start_tts_usage_metrics(text) + yield TTSStartedFrame() + self._request_id = str(uuid.uuid4()) + + # Send the text + text_message = { + "event": "text", + "text": text, + } + try: + await self._get_websocket().send(ormsgpack.packb(text_message)) + await self.start_tts_usage_metrics(text) + + # Send flush event to force audio generation + flush_message = {"event": "flush"} + await self._get_websocket().send(ormsgpack.packb(flush_message)) + except Exception as e: + logger.error(f"{self} error sending message: {e}") + yield TTSStoppedFrame() + await self._disconnect() + await self._connect() + + yield None + + except Exception as e: + logger.error(f"Error generating TTS: {e}") + yield ErrorFrame(f"Error: {str(e)}") diff --git a/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py b/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py index 953137106..95642b19d 100644 --- a/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py +++ b/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index 24e2b015e..0d5bc802f 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -1,18 +1,17 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # # import base64 -import json import io - -from pydantic import BaseModel, Field +import json from typing import List, Literal, Optional from PIL import Image +from pydantic import BaseModel, Field from pipecat.frames.frames import ImageRawFrame diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index bf433054a..dd4375486 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -230,7 +230,6 @@ class GeminiMultimodalLiveLLMService(LLMService): async def start(self, frame: StartFrame): await super().start(frame) - await self._connect() async def stop(self, frame: EndFrame): await super().stop(frame) @@ -434,8 +433,9 @@ class GeminiMultimodalLiveLLMService(LLMService): async def _ws_send(self, message): # logger.debug(f"Sending message to websocket: {message}") try: - if self._websocket: - await self._websocket.send(json.dumps(message)) + if not self._websocket: + await self._connect() + await self._websocket.send(json.dumps(message)) except Exception as e: if self._disconnecting: return diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index 8909c4bb2..33f8b0cf1 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 5442ee91c..b52c1f6dc 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -23,6 +23,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMUpdateSettingsFrame, + OpenAILLMContextAssistantTimestampFrame, TextFrame, TTSAudioRawFrame, TTSStartedFrame, @@ -41,6 +42,7 @@ from pipecat.services.openai import ( OpenAIUserContextAggregator, ) from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 try: import google.ai.generativelanguage as glm @@ -227,6 +229,7 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator): # if the tasks gets cancelled we won't be able to clear things up. self._aggregation = "" + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) @@ -300,9 +303,14 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if run_llm: await self._user_context_aggregator.push_context_frame() + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + except Exception as e: logger.exception(f"Error processing frame: {e}") @@ -412,6 +420,25 @@ class GoogleLLMContext(OpenAILLMContext): # self.add_message(message) def from_standard_message(self, message): + """Convert standard format message to Google Content object. + + Handles conversion of text, images, and function calls to Google's format. + System messages are stored separately and return None. + + Args: + message: Message in standard format: + { + "role": "user/assistant/system/tool", + "content": str | [{"type": "text/image_url", ...}] | None, + "tool_calls": [{"function": {"name": str, "arguments": str}}] + } + + Returns: + glm.Content object with: + - role: "user" or "model" (converted from "assistant") + - parts: List[Part] containing text, inline_data, or function calls + Returns None for system messages. + """ role = message["role"] content = message.get("content", []) if role == "system": @@ -461,6 +488,27 @@ class GoogleLLMContext(OpenAILLMContext): return message def to_standard_messages(self, obj) -> list: + """Convert Google Content object to standard structured format. + + Handles text, images, and function calls from Google's Content/Part objects. + + Args: + obj: Google Content object with: + - role: "model" (converted to "assistant") or "user" + - parts: List[Part] containing text, inline_data, or function calls + + Returns: + List of messages in standard format: + [ + { + "role": "user/assistant/tool", + "content": [ + {"type": "text", "text": str} | + {"type": "image_url", "image_url": {"url": str}} + ] + } + ] + """ msg = {"role": obj.role, "content": []} if msg["role"] == "model": msg["role"] = "assistant" @@ -545,6 +593,8 @@ class GoogleLLMService(LLMService): model: str = "gemini-1.5-flash-latest", params: InputParams = InputParams(), system_instruction: Optional[str] = None, + tools: Optional[List[Dict[str, Any]]] = None, + tool_config: Optional[Dict[str, Any]] = None, **kwargs, ): super().__init__(**kwargs) @@ -559,6 +609,8 @@ class GoogleLLMService(LLMService): "top_p": params.top_p, "extra": params.extra if isinstance(params.extra, dict) else {}, } + self._tools = tools + self._tool_config = tool_config def can_generate_metrics(self) -> bool: return True @@ -577,7 +629,8 @@ class GoogleLLMService(LLMService): try: logger.debug( - f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}" + # f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}" + f"Generating chat: {context.get_messages_for_logging()}" ) messages = context.messages @@ -601,28 +654,41 @@ class GoogleLLMService(LLMService): generation_config = GenerationConfig(**generation_params) if generation_params else None await self.start_ttfb_metrics() - tools = context.tools if context.tools else [] + tools = [] + if context.tools: + tools = context.tools + elif self._tools: + tools = self._tools + tool_config = None + if self._tool_config: + tool_config = self._tool_config response = await self._client.generate_content_async( - contents=messages, tools=tools, stream=True, generation_config=generation_config + contents=messages, + tools=tools, + stream=True, + generation_config=generation_config, + tool_config=tool_config, ) await self.stop_ttfb_metrics() if response.usage_metadata: + # Use only the prompt token count from the response object prompt_tokens = response.usage_metadata.prompt_token_count - completion_tokens = response.usage_metadata.candidates_token_count - total_tokens = response.usage_metadata.total_token_count + total_tokens = prompt_tokens async for chunk in response: if chunk.usage_metadata: - prompt_tokens += response.usage_metadata.prompt_token_count - completion_tokens += response.usage_metadata.candidates_token_count - total_tokens += response.usage_metadata.total_token_count + # Use only the completion_tokens from the chunks. Prompt tokens are already counted and + # are repeated here. + completion_tokens += chunk.usage_metadata.candidates_token_count + total_tokens += chunk.usage_metadata.candidates_token_count try: for c in chunk.parts: if c.text: await self.push_frame(TextFrame(c.text)) elif c.function_call: + logger.debug(f"!!! Function call: {c.function_call}") args = type(c.function_call).to_dict(c.function_call).get("args", {}) await self.call_function( context=context, diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index 505dfcca5..f38d86966 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -1,15 +1,106 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # +import json +from dataclasses import dataclass + from loguru import logger from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.openai import OpenAILLMService +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.services.openai import ( + OpenAIAssistantContextAggregator, + OpenAILLMService, + OpenAIUserContextAggregator, +) + + +class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Custom assistant context aggregator for Grok that handles empty content requirement.""" + + async def _push_aggregation(self): + if not ( + self._aggregation or self._function_call_result or self._pending_image_frame_message + ): + return + + run_llm = False + + aggregation = self._aggregation + self._reset() + + try: + if self._function_call_result: + frame = self._function_call_result + self._function_call_result = None + if frame.result: + # Grok requires an empty content field for function calls + self._context.add_message( + { + "role": "assistant", + "content": "", # Required by Grok + "tool_calls": [ + { + "id": frame.tool_call_id, + "function": { + "name": frame.function_name, + "arguments": json.dumps(frame.arguments), + }, + "type": "function", + } + ], + } + ) + self._context.add_message( + { + "role": "tool", + "content": json.dumps(frame.result), + "tool_call_id": frame.tool_call_id, + } + ) + # Only run the LLM if there are no more function calls in progress. + run_llm = not bool(self._function_calls_in_progress) + else: + self._context.add_message({"role": "assistant", "content": aggregation}) + + if self._pending_image_frame_message: + frame = self._pending_image_frame_message + self._pending_image_frame_message = None + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) + run_llm = True + + if run_llm: + await self._user_context_aggregator.push_context_frame() + + frame = OpenAILLMContextFrame(self._context) + await self.push_frame(frame) + + except Exception as e: + logger.error(f"Error processing frame: {e}") + + +@dataclass +class GrokContextAggregatorPair: + _user: "OpenAIUserContextAggregator" + _assistant: "GrokAssistantContextAggregator" + + def user(self) -> "OpenAIUserContextAggregator": + return self._user + + def assistant(self) -> "GrokAssistantContextAggregator": + return self._assistant class GrokLLMService(OpenAILLMService): @@ -101,3 +192,13 @@ class GrokLLMService(OpenAILLMService): # Update completion tokens count if it has increased if tokens.completion_tokens > self._completion_tokens: self._completion_tokens = tokens.completion_tokens + + @staticmethod + def create_context_aggregator( + context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + ) -> GrokContextAggregatorPair: + user = OpenAIUserContextAggregator(context) + assistant = GrokAssistantContextAggregator( + user, expect_stripped_words=assistant_expect_stripped_words + ) + return GrokContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index c9b49d636..f035f7607 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 04223a1a1..ba931dc38 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,6 +8,7 @@ import asyncio from typing import AsyncGenerator from loguru import logger +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from pipecat.frames.frames import ( CancelFrame, @@ -159,31 +160,47 @@ class LmntTTSService(TTSService): except Exception as e: logger.error(f"{self} error closing connection: {e}") + async def _receive_messages(self): + async for msg in self._connection: + if "error" in msg: + logger.error(f'{self} error: {msg["error"]}') + await self.push_frame(TTSStoppedFrame()) + await self.stop_all_metrics() + await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) + elif "audio" in msg: + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame( + audio=msg["audio"], + sample_rate=self._settings["output_format"]["sample_rate"], + num_channels=1, + ) + await self.push_frame(frame) + else: + logger.error(f"{self}: LMNT error, unknown message type: {msg}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_lmnt() + await self._connect_lmnt() + async def _receive_task_handler(self): while True: try: - async for msg in self._connection: - if "error" in msg: - logger.error(f'{self} error: {msg["error"]}') - await self.push_frame(TTSStoppedFrame()) - await self.stop_all_metrics() - await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) - elif "audio" in msg: - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame( - audio=msg["audio"], - sample_rate=self._settings["output_format"]["sample_rate"], - num_channels=1, - ) - await self.push_frame(frame) - else: - logger.error(f"{self}: LMNT error, unknown message type: {msg}") + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception: {e}") - await self._disconnect_lmnt() - await self._connect_lmnt() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"Generating TTS: [{text}]") diff --git a/src/pipecat/services/moondream.py b/src/pipecat/services/moondream.py index 74442dfee..eebc12ce6 100644 --- a/src/pipecat/services/moondream.py +++ b/src/pipecat/services/moondream.py @@ -1,23 +1,20 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - -from PIL import Image - from typing import AsyncGenerator +from loguru import logger +from PIL import Image + from pipecat.frames.frames import ErrorFrame, Frame, TextFrame, VisionImageRawFrame from pipecat.services.ai_services import VisionService -from loguru import logger - try: import torch - from transformers import AutoModelForCausalLM, AutoTokenizer except ModuleNotFoundError as e: logger.error(f"Exception: {e}") @@ -26,9 +23,7 @@ except ModuleNotFoundError as e: def detect_device(): - """ - Detects the appropriate device to run on, and return the device and dtype. - """ + """Detects the appropriate device to run on, and return the device and dtype.""" try: import intel_extension_for_pytorch diff --git a/src/pipecat/services/nim.py b/src/pipecat/services/nim.py index 2b57a5047..3250ba420 100644 --- a/src/pipecat/services/nim.py +++ b/src/pipecat/services/nim.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/ollama.py b/src/pipecat/services/ollama.py index 0a6a4ce6a..1e74b303e 100644 --- a/src/pipecat/services/ollama.py +++ b/src/pipecat/services/ollama.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 43ad16536..f614ec575 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -25,6 +25,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMUpdateSettingsFrame, + OpenAILLMContextAssistantTimestampFrame, StartInterruptionFrame, TextFrame, TTSAudioRawFrame, @@ -46,6 +47,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import ImageGenService, LLMService, TTSService +from pipecat.utils.time import time_now_iso8601 try: from openai import ( @@ -559,7 +561,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): self._context.add_message( { "role": "assistant", - "content": "", # content field required for Grok function calling "tool_calls": [ { "id": frame.tool_call_id, @@ -598,8 +599,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): if run_llm: await self._user_context_aggregator.push_context_frame() + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + except Exception as e: logger.error(f"Error processing frame: {e}") diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 2b6ff968f..049195188 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -21,7 +21,7 @@ from pipecat.services.openai import ( ) from . import events -from .frames import RealtimeMessagesUpdateFrame, RealtimeFunctionCallResultFrame +from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame class OpenAIRealtimeLLMContext(OpenAILLMContext): diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 0515012e3..f757f8f74 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai_realtime_beta/frames.py b/src/pipecat/services/openai_realtime_beta/frames.py index 54bdcd467..22fa7e87c 100644 --- a/src/pipecat/services/openai_realtime_beta/frames.py +++ b/src/pipecat/services/openai_realtime_beta/frames.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index ac492a205..ea5d4c41e 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,10 +8,10 @@ import asyncio import base64 import json import time - from dataclasses import dataclass import websockets +from loguru import logger from pipecat.frames.frames import ( BotStoppedSpeakingFrame, @@ -48,13 +48,11 @@ from pipecat.utils.time import time_now_iso8601 from . import events from .context import ( + OpenAIRealtimeAssistantContextAggregator, OpenAIRealtimeLLMContext, OpenAIRealtimeUserContextAggregator, - OpenAIRealtimeAssistantContextAggregator, ) -from .frames import RealtimeMessagesUpdateFrame, RealtimeFunctionCallResultFrame - -from loguru import logger +from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame @dataclass @@ -74,15 +72,17 @@ class OpenAIRealtimeBetaLLMService(LLMService): self, *, api_key: str, - base_url="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01", + model: str = "gpt-4o-realtime-preview-2024-12-17", + base_url: str = "wss://api.openai.com/v1/realtime", session_properties: events.SessionProperties = events.SessionProperties(), start_audio_paused: bool = False, send_transcription_frames: bool = True, **kwargs, ): - super().__init__(base_url=base_url, **kwargs) + full_url = f"{base_url}?model={model}" + super().__init__(base_url=full_url, **kwargs) self.api_key = api_key - self.base_url = base_url + self.base_url = full_url self._session_properties: events.SessionProperties = session_properties self._audio_input_paused = start_audio_paused diff --git a/src/pipecat/services/openpipe.py b/src/pipecat/services/openpipe.py index 827ffcb85..ea36c9129 100644 --- a/src/pipecat/services/openpipe.py +++ b/src/pipecat/services/openpipe.py @@ -1,19 +1,20 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # from typing import Dict, List +from loguru import logger + from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService -from loguru import logger - try: - from openpipe import AsyncOpenAI as OpenPipeAI, AsyncStream - from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk + from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam + from openpipe import AsyncOpenAI as OpenPipeAI + from openpipe import AsyncStream except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 00c5d4d34..ddbcd76b1 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -15,6 +15,7 @@ import aiohttp import websockets from loguru import logger from pydantic import BaseModel +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from pipecat.frames.frames import ( BotStoppedSpeakingFrame, @@ -208,7 +209,7 @@ class PlayHTTTSService(TTSService): async def _get_websocket_url(self): async with aiohttp.ClientSession() as session: async with session.post( - "https://api.play.ht/api/v3/websocket-auth", + "https://api.play.ht/api/v4/websocket-auth", headers={ "Authorization": f"Bearer {self._api_key}", "X-User-Id": self._user_id, @@ -217,10 +218,19 @@ class PlayHTTTSService(TTSService): ) as response: if response.status in (200, 201): data = await response.json() - if "websocket_url" in data and isinstance(data["websocket_url"], str): - self._websocket_url = data["websocket_url"] + # Handle the new response format with multiple URLs + if "websocket_urls" in data: + # Select URL based on voice_engine + if self._settings["voice_engine"] in data["websocket_urls"]: + self._websocket_url = data["websocket_urls"][ + self._settings["voice_engine"] + ] + else: + raise ValueError( + f"Unsupported voice engine: {self._settings['voice_engine']}" + ) else: - raise ValueError("Invalid or missing WebSocket URL in response") + raise ValueError("Invalid response: missing websocket_urls") else: raise Exception(f"Failed to get WebSocket URL: {response.status}") @@ -234,35 +244,56 @@ class PlayHTTTSService(TTSService): await self.stop_all_metrics() self._request_id = None + async def _receive_messages(self): + async for message in self._get_websocket(): + if isinstance(message, bytes): + # Skip the WAV header message + if message.startswith(b"RIFF"): + continue + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame(message, self._settings["sample_rate"], 1) + await self.push_frame(frame) + else: + logger.debug(f"Received text message: {message}") + try: + msg = json.loads(message) + if msg.get("type") == "start": + # Handle start of stream + logger.debug(f"Started processing request: {msg.get('request_id')}") + elif msg.get("type") == "end": + # Handle end of stream + if "request_id" in msg and msg["request_id"] == self._request_id: + await self.push_frame(TTSStoppedFrame()) + self._request_id = None + elif "error" in msg: + logger.error(f"{self} error: {msg}") + await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) + except json.JSONDecodeError: + logger.error(f"Invalid JSON message: {message}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + async def _receive_task_handler(self): while True: try: - async for message in self._get_websocket(): - if isinstance(message, bytes): - # Skip the WAV header message - if message.startswith(b"RIFF"): - continue - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame(message, self._settings["sample_rate"], 1) - await self.push_frame(frame) - else: - logger.debug(f"Received text message: {message}") - try: - msg = json.loads(message) - if "request_id" in msg and msg["request_id"] == self._request_id: - await self.push_frame(TTSStoppedFrame()) - self._request_id = None - elif "error" in msg: - logger.error(f"{self} error: {msg}") - await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) - except json.JSONDecodeError: - logger.error(f"Invalid JSON message: {message}") + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception in receive task: {e}") - await self._disconnect_websocket() - await self._connect_websocket() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 6cf2c8129..4bfa56b20 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + from typing import AsyncGenerator, Optional import aiohttp diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index f57372775..77396f84a 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,7 +8,7 @@ import asyncio from typing import AsyncGenerator, Optional from loguru import logger -from pydantic.main import BaseModel +from pydantic import BaseModel from pipecat.frames.frames import ( CancelFrame, @@ -35,6 +35,8 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") +FASTPITCH_TIMEOUT_SECS = 5 + class FastPitchTTSService(TTSService): class InputParams(BaseModel): @@ -76,7 +78,10 @@ class FastPitchTTSService(TTSService): ) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - def read_audio_responses(): + def read_audio_responses(queue: asyncio.Queue): + def add_response(r): + asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop()) + try: responses = self._service.synthesize_online( text, @@ -87,26 +92,35 @@ class FastPitchTTSService(TTSService): quality=self._quality, custom_dictionary={}, ) - return responses + for r in responses: + add_response(r) + add_response(None) except Exception as e: logger.error(f"{self} exception: {e}") - return [] + add_response(None) await self.start_ttfb_metrics() yield TTSStartedFrame() logger.debug(f"Generating TTS: [{text}]") - responses = await asyncio.to_thread(read_audio_responses) - for resp in responses: - await self.stop_ttfb_metrics() + try: + queue = asyncio.Queue() + await asyncio.to_thread(read_audio_responses, queue) - frame = TTSAudioRawFrame( - audio=resp.audio, - sample_rate=self._sample_rate, - num_channels=1, - ) - yield frame + # Wait for the thread to start. + resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS) + while resp: + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame( + audio=resp.audio, + sample_rate=self._sample_rate, + num_channels=1, + ) + yield frame + resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS) + except asyncio.TimeoutError: + logger.error(f"{self} timeout waiting for audio response") await self.start_tts_usage_metrics(text) yield TTSStoppedFrame() diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index bfae861dc..19825daf3 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -1,29 +1,27 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio +import numpy as np +from loguru import logger + from pipecat.frames.frames import ( + CancelFrame, + EndFrame, Frame, OutputImageRawFrame, - TTSAudioRawFrame, StartInterruptionFrame, - EndFrame, - CancelFrame, + TTSAudioRawFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, StartFrame -import numpy as np - -from loguru import logger - try: from av.audio.frame import AudioFrame from av.audio.resampler import AudioResampler - from simli import SimliClient, SimliConfig except ModuleNotFoundError as e: logger.error(f"Exception: {e}") diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index ff2b7fb87..df41a9fc9 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -7,24 +7,24 @@ """This module implements Tavus as a sink transport layer""" -import aiohttp import base64 +import aiohttp +from loguru import logger + +from pipecat.audio.utils import resample_audio from pipecat.frames.frames import ( + CancelFrame, + EndFrame, Frame, - TTSAudioRawFrame, + StartInterruptionFrame, TransportMessageUrgentFrame, + TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, - StartInterruptionFrame, - EndFrame, - CancelFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AIService -from pipecat.audio.utils import resample_audio - -from loguru import logger class TavusVideoService(AIService): diff --git a/src/pipecat/services/to_be_updated/cloudflare_ai_service.py b/src/pipecat/services/to_be_updated/cloudflare_ai_service.py index 1329f9c79..ff637ff1a 100644 --- a/src/pipecat/services/to_be_updated/cloudflare_ai_service.py +++ b/src/pipecat/services/to_be_updated/cloudflare_ai_service.py @@ -1,5 +1,6 @@ -import requests import os + +import requests from services.ai_service import AIService # Note that Cloudflare's AI workers are still in beta. diff --git a/src/pipecat/services/to_be_updated/google_ai_service.py b/src/pipecat/services/to_be_updated/google_ai_service.py index 25668ca0a..3ca688750 100644 --- a/src/pipecat/services/to_be_updated/google_ai_service.py +++ b/src/pipecat/services/to_be_updated/google_ai_service.py @@ -1,11 +1,12 @@ -from services.ai_service import AIService -import openai import os +import openai + # To use Google Cloud's AI products, you'll need to install Google Cloud # CLI and enable the TTS and in your project: # https://cloud.google.com/sdk/docs/install from google.cloud import texttospeech +from services.ai_service import AIService class GoogleAIService(AIService): diff --git a/src/pipecat/services/to_be_updated/mock_ai_service.py b/src/pipecat/services/to_be_updated/mock_ai_service.py index dc200f622..0825cde33 100644 --- a/src/pipecat/services/to_be_updated/mock_ai_service.py +++ b/src/pipecat/services/to_be_updated/mock_ai_service.py @@ -1,6 +1,7 @@ import io -import requests import time + +import requests from PIL import Image from services.ai_service import AIService diff --git a/src/pipecat/services/together.py b/src/pipecat/services/together.py index e18e9650d..e43c7a104 100644 --- a/src/pipecat/services/together.py +++ b/src/pipecat/services/together.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/whisper.py b/src/pipecat/services/whisper.py index a4635c6cb..6cbcf4793 100644 --- a/src/pipecat/services/whisper.py +++ b/src/pipecat/services/whisper.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -7,18 +7,16 @@ """This module implements Whisper transcription with a locally-downloaded model.""" import asyncio - from enum import Enum from typing import AsyncGenerator import numpy as np +from loguru import logger from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame from pipecat.services.ai_services import SegmentedSTTService from pipecat.utils.time import time_now_iso8601 -from loguru import logger - try: from faster_whisper import WhisperModel except ModuleNotFoundError as e: @@ -63,7 +61,8 @@ class WhisperSTTService(SegmentedSTTService): def _load(self): """Loads the Whisper model. Note that if this is the first time - this model is being run, it will take time to download.""" + this model is being run, it will take time to download. + """ logger.debug("Loading Whisper model...") self._model = WhisperModel( self.model_name, device=self._device, compute_type=self._compute_type diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index 900701ffa..47b91967e 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/sync/base_notifier.py b/src/pipecat/sync/base_notifier.py index c7770ab26..757c1326b 100644 --- a/src/pipecat/sync/base_notifier.py +++ b/src/pipecat/sync/base_notifier.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/sync/event_notifier.py b/src/pipecat/sync/event_notifier.py index f02dcbdae..ac87419da 100644 --- a/src/pipecat/sync/event_notifier.py +++ b/src/pipecat/sync/event_notifier.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index ee16c14a7..0bba6b2f1 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 025a5bed2..165e9ca4b 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index b25b4c78c..8f2715b5c 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -15,7 +15,6 @@ from PIL import Image from pipecat.audio.vad.vad_analyzer import VAD_STOP_SECS from pipecat.frames.frames import ( - AudioRawFrame, BotSpeakingFrame, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, @@ -30,9 +29,9 @@ from pipecat.frames.frames import ( StartInterruptionFrame, StopInterruptionFrame, SystemFrame, - TTSAudioRawFrame, TransportMessageFrame, TransportMessageUrgentFrame, + TTSAudioRawFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.base_transport import TransportParams @@ -52,9 +51,7 @@ class BaseOutputTransport(FrameProcessor): self._sink_clock_task = None # Task to write/send audio and image frames. - self._audio_out_task = None self._camera_out_task = None - self._running_out_tasks = True # These are the images that we should send to the camera at our desired # framerate. @@ -77,22 +74,31 @@ class BaseOutputTransport(FrameProcessor): # Start audio mixer. if self._params.audio_out_mixer: await self._params.audio_out_mixer.start(self._params.audio_out_sample_rate) - self._create_output_tasks() + self._create_camera_task() self._create_sink_tasks() async def stop(self, frame: EndFrame): - # We can't cancel output tasks because there might still be audio - # buffered to be played. - await self._stop_output_tasks() - # Stop audio mixer. - if self._params.audio_out_mixer: - await self._params.audio_out_mixer.stop() + # Let the sink tasks process the queue until they reach this EndFrame. + await self._sink_clock_queue.put((sys.maxsize, frame.id, frame)) + await self._sink_queue.put(frame) + + # At this point we have enqueued an EndFrame and we need to wait for + # that EndFrame to be processed by the sink tasks. We also need to wait + # for these tasks before cancelling the camera and audio tasks below + # because they might be still rendering. + if self._sink_task: + await self._sink_task + if self._sink_clock_task: + await self._sink_clock_task + + # We can now cancel the camera task. + await self._cancel_camera_task() async def cancel(self, frame: CancelFrame): # Since we are cancelling everything it doesn't matter if we cancel sink # tasks first or not. await self._cancel_sink_tasks() - await self._cancel_output_tasks() + await self._cancel_camera_task() async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): pass @@ -103,6 +109,12 @@ class BaseOutputTransport(FrameProcessor): async def write_raw_audio_frames(self, frames: bytes): pass + async def send_audio(self, frame: OutputAudioRawFrame): + await self.queue_frame(frame, FrameDirection.DOWNSTREAM) + + async def send_image(self, frame: OutputImageRawFrame | SpriteFrame): + await self.queue_frame(frame, FrameDirection.DOWNSTREAM) + # # Frame processor # @@ -132,11 +144,8 @@ class BaseOutputTransport(FrameProcessor): await self.push_frame(frame, direction) # Control frames. elif isinstance(frame, EndFrame): - # Process sink tasks. - await self._stop_sink_tasks(frame) - # Now we can stop. await self.stop(frame) - # We finally push EndFrame down so PipelineTask stops nicely. + # Keep pushing EndFrame down so all the pipeline stops nicely. await self.push_frame(frame, direction) elif isinstance(frame, MixerControlFrame) and self._params.audio_out_mixer: await self._params.audio_out_mixer.process_frame(frame) @@ -151,30 +160,16 @@ class BaseOutputTransport(FrameProcessor): else: await self._sink_queue.put(frame) - async def _stop_sink_tasks(self, frame: EndFrame): - # Let the sink tasks process the queue until they reach this EndFrame. - await self._sink_clock_queue.put((sys.maxsize, frame.id, frame)) - await self._sink_queue.put(frame) - - # At this point we have enqueued an EndFrame and we need to wait for - # that EndFrame to be processed by the sink tasks. We also need to wait - # for these tasks before cancelling the camera and audio tasks below - # because they might be still rendering. - if self._sink_task: - await self._sink_task - if self._sink_clock_task: - await self._sink_clock_task - async def _handle_interruptions(self, frame: Frame): if not self.interruptions_allowed: return if isinstance(frame, StartInterruptionFrame): - # Cancel sink and output tasks. + # Cancel sink and camera tasks. await self._cancel_sink_tasks() - await self._cancel_output_tasks() - # Create sink and output tasks. - self._create_output_tasks() + await self._cancel_camera_task() + # Create sink and camera tasks. + self._create_camera_task() self._create_sink_tasks() # Let's send a bot stopped speaking if we have to. await self._bot_stopped_speaking() @@ -183,19 +178,16 @@ class BaseOutputTransport(FrameProcessor): if not self._params.audio_out_enabled: return - if self._params.audio_out_is_live: - await self._audio_out_queue.put(frame) - else: - cls = type(frame) - self._audio_buffer.extend(frame.audio) - while len(self._audio_buffer) >= self._audio_chunk_size: - chunk = cls( - bytes(self._audio_buffer[: self._audio_chunk_size]), - sample_rate=frame.sample_rate, - num_channels=frame.num_channels, - ) - await self._sink_queue.put(chunk) - self._audio_buffer = self._audio_buffer[self._audio_chunk_size :] + cls = type(frame) + self._audio_buffer.extend(frame.audio) + while len(self._audio_buffer) >= self._audio_chunk_size: + chunk = cls( + bytes(self._audio_buffer[: self._audio_chunk_size]), + sample_rate=frame.sample_rate, + num_channels=frame.num_channels, + ) + await self._sink_queue.put(chunk) + self._audio_buffer = self._audio_buffer[self._audio_chunk_size :] async def _handle_image(self, frame: OutputImageRawFrame | SpriteFrame): if not self._params.camera_out_enabled: @@ -244,30 +236,12 @@ class BaseOutputTransport(FrameProcessor): self._sink_clock_task = None async def _sink_frame_handler(self, frame: Frame): - if isinstance(frame, OutputAudioRawFrame): - await self._audio_out_queue.put(frame) - elif isinstance(frame, OutputImageRawFrame): + if isinstance(frame, OutputImageRawFrame): await self._set_camera_image(frame) elif isinstance(frame, SpriteFrame): await self._set_camera_images(frame.images) elif isinstance(frame, TransportMessageFrame): await self.send_message(frame) - # We will push EndFrame later. - elif not isinstance(frame, EndFrame): - await self.push_frame(frame) - - async def _sink_task_handler(self): - running = True - while running: - try: - frame = await self._sink_queue.get() - await self._sink_frame_handler(frame) - running = not isinstance(frame, EndFrame) - self._sink_queue.task_done() - except asyncio.CancelledError: - break - except Exception as e: - logger.exception(f"{self} error processing sink queue: {e}") async def _sink_clock_task_handler(self): running = True @@ -286,56 +260,107 @@ class BaseOutputTransport(FrameProcessor): if timestamp > current_time: wait_time = nanoseconds_to_seconds(timestamp - current_time) await asyncio.sleep(wait_time) + + # Handle frame. await self._sink_frame_handler(frame) + # Also, push frame downstream in case anyone else needs it. + await self.push_frame(frame) + self._sink_clock_queue.task_done() except asyncio.CancelledError: break except Exception as e: logger.exception(f"{self} error processing sink clock queue: {e}") + def _next_frame(self) -> AsyncGenerator[Frame, None]: + async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]: + while True: + try: + frame = await asyncio.wait_for(self._sink_queue.get(), timeout=vad_stop_secs) + yield frame + except asyncio.TimeoutError: + # Notify the bot stopped speaking upstream if necessary. + await self._bot_stopped_speaking() + + async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]: + last_frame_time = 0 + silence = b"\x00" * self._audio_chunk_size + while True: + try: + frame = self._sink_queue.get_nowait() + if isinstance(frame, OutputAudioRawFrame): + frame.audio = await self._params.audio_out_mixer.mix(frame.audio) + last_frame_time = time.time() + yield frame + except asyncio.QueueEmpty: + # Notify the bot stopped speaking upstream if necessary. + diff_time = time.time() - last_frame_time + if diff_time > vad_stop_secs: + await self._bot_stopped_speaking() + # Generate an audio frame with only the mixer's part. + frame = OutputAudioRawFrame( + audio=await self._params.audio_out_mixer.mix(silence), + sample_rate=self._params.audio_out_sample_rate, + num_channels=self._params.audio_out_channels, + ) + yield frame + + vad_stop_secs = ( + self._params.vad_analyzer.params.stop_secs + if self._params.vad_analyzer + else VAD_STOP_SECS + ) + if self._params.audio_out_mixer: + return with_mixer(vad_stop_secs) + else: + return without_mixer(vad_stop_secs) + + async def _sink_task_handler(self): + try: + async for frame in self._next_frame(): + # Notify the bot started speaking upstream if necessary and that + # it's actually speaking. + if isinstance(frame, TTSAudioRawFrame): + await self._bot_started_speaking() + await self.push_frame(BotSpeakingFrame()) + await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) + + # No need to push EndFrame, it's pushed from process_frame(). + if isinstance(frame, EndFrame): + break + + # Handle frame. + await self._sink_frame_handler(frame) + + # Also, push frame downstream in case anyone else needs it. + await self.push_frame(frame) + + # Send audio. + if isinstance(frame, OutputAudioRawFrame): + await self.write_raw_audio_frames(frame.audio) + except asyncio.CancelledError: + pass + except Exception as e: + logger.exception(f"{self} error writing to microphone: {e}") + # - # Output tasks + # Camera task # - def _create_output_tasks(self): + def _create_camera_task(self): loop = self.get_event_loop() # Create camera output queue and task if needed. if self._params.camera_out_enabled: self._camera_out_queue = asyncio.Queue() self._camera_out_task = loop.create_task(self._camera_out_task_handler()) - # Create audio output queue and task if needed. - if self._params.audio_out_enabled: - self._audio_out_queue = asyncio.Queue() - self._audio_out_task = loop.create_task(self._audio_out_task_handler()) - async def _stop_output_tasks(self): - self._running_out_tasks = False - # Stop camera output task. - if self._camera_out_task and self._params.camera_out_enabled: - await self._camera_out_task - # Stop audio output task. - if self._audio_out_task and self._params.audio_out_enabled: - await self._audio_out_task - - async def _cancel_output_tasks(self): + async def _cancel_camera_task(self): # Stop camera output task. if self._camera_out_task and self._params.camera_out_enabled: self._camera_out_task.cancel() await self._camera_out_task self._camera_out_task = None - # Stop audio output task. - if self._audio_out_task and self._params.audio_out_enabled: - self._audio_out_task.cancel() - await self._audio_out_task - self._audio_out_task = None - - # - # Camera out - # - - async def send_image(self, frame: OutputImageRawFrame | SpriteFrame): - await self.queue_frame(frame, FrameDirection.DOWNSTREAM) async def _draw_image(self, frame: OutputImageRawFrame): desired_size = (self._params.camera_out_width, self._params.camera_out_height) @@ -361,7 +386,7 @@ class BaseOutputTransport(FrameProcessor): self._camera_out_frame_index = 0 self._camera_out_frame_duration = 1 / self._params.camera_out_framerate self._camera_out_frame_reset = self._camera_out_frame_duration * 5 - while self._running_out_tasks: + while True: try: if self._params.camera_out_is_live: await self._camera_out_is_live_handler() @@ -400,74 +425,3 @@ class BaseOutputTransport(FrameProcessor): await self._draw_image(image) self._camera_out_queue.task_done() - - # - # Audio out - # - - async def send_audio(self, frame: OutputAudioRawFrame): - await self.queue_frame(frame, FrameDirection.DOWNSTREAM) - - def _next_audio_frame(self) -> AsyncGenerator[AudioRawFrame, None]: - async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[AudioRawFrame, None]: - while self._running_out_tasks or self._bot_speaking: - try: - frame = await asyncio.wait_for( - self._audio_out_queue.get(), timeout=vad_stop_secs - ) - yield frame - except asyncio.TimeoutError: - # Notify the bot stopped speaking upstream if necessary. - await self._bot_stopped_speaking() - - async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[AudioRawFrame, None]: - last_frame_time = 0 - silence = b"\x00" * self._audio_chunk_size - while self._running_out_tasks or self._bot_speaking: - try: - frame = self._audio_out_queue.get_nowait() - frame.audio = await self._params.audio_out_mixer.mix(frame.audio) - last_frame_time = time.time() - yield frame - except asyncio.QueueEmpty: - # Notify the bot stopped speaking upstream if necessary. - diff_time = time.time() - last_frame_time - if diff_time > vad_stop_secs: - await self._bot_stopped_speaking() - # Generate an audio frame with only the mixer's part. - frame = OutputAudioRawFrame( - audio=await self._params.audio_out_mixer.mix(silence), - sample_rate=self._params.audio_out_sample_rate, - num_channels=self._params.audio_out_channels, - ) - yield frame - - vad_stop_secs = ( - self._params.vad_analyzer.params.stop_secs - if self._params.vad_analyzer - else VAD_STOP_SECS - ) - if self._params.audio_out_mixer: - return with_mixer(vad_stop_secs) - else: - return without_mixer(vad_stop_secs) - - async def _audio_out_task_handler(self): - try: - async for frame in self._next_audio_frame(): - # Notify the bot started speaking upstream if necessary and that - # it's actually speaking. - if isinstance(frame, TTSAudioRawFrame): - await self._bot_started_speaking() - await self.push_frame(BotSpeakingFrame()) - await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) - - # Also, push frame downstream in case anyone else needs it. - await self.push_frame(frame) - - # Send audio. - await self.write_raw_audio_frames(frame.audio) - except asyncio.CancelledError: - pass - except Exception as e: - logger.exception(f"{self} error writing to microphone: {e}") diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index 91f95fbbc..8d997ab08 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -1,25 +1,22 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio import inspect - from abc import ABC, abstractmethod from typing import Optional -from pydantic import ConfigDict -from pydantic import BaseModel +from loguru import logger +from pydantic import BaseModel, ConfigDict from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer from pipecat.audio.vad.vad_analyzer import VADAnalyzer from pipecat.processors.frame_processor import FrameProcessor -from loguru import logger - class TransportParams(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index e1ccefec2..4e03701d8 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -1,21 +1,20 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - from concurrent.futures import ThreadPoolExecutor +from loguru import logger + from pipecat.frames.frames import InputAudioRawFrame, StartFrame from pipecat.processors.frame_processor import FrameProcessor from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: import pyaudio except ModuleNotFoundError as e: diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index ed7cdbea6..98ad908c7 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -1,23 +1,21 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import asyncio - +import tkinter as tk from concurrent.futures import ThreadPoolExecutor import numpy as np -import tkinter as tk +from loguru import logger from pipecat.frames.frames import InputAudioRawFrame, OutputImageRawFrame, StartFrame from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: import pyaudio except ModuleNotFoundError as e: diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index cd21fbe0d..02270d5c0 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -10,8 +10,9 @@ import io import time import typing import wave - from typing import Awaitable, Callable + +from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( @@ -27,8 +28,6 @@ from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: from fastapi import WebSocket from starlette.websockets import WebSocketState diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 711bc7596..4bbfeef7e 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,8 +8,9 @@ import asyncio import io import time import wave - from typing import Awaitable, Callable + +from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( @@ -28,8 +29,6 @@ from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: import websockets except ModuleNotFoundError as e: @@ -69,8 +68,8 @@ class WebsocketServerInputTransport(BaseInputTransport): self._stop_server_event = asyncio.Event() async def start(self, frame: StartFrame): - self._server_task = self.get_event_loop().create_task(self._server_task_handler()) await super().start(frame) + self._server_task = self.get_event_loop().create_task(self._server_task_handler()) async def stop(self, frame: EndFrame): await super().stop(frame) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 7456ef816..ae43f0efa 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -491,6 +491,21 @@ class DailyTransportClient(EventHandler): self._client.stop_dialout(participant_id, completion=completion_callback(future)) await future + async def send_dtmf(self, settings): + future = self._loop.create_future() + self._client.send_dtmf(settings, completion=completion_callback(future)) + await future + + async def sip_call_transfer(self, settings): + future = self._loop.create_future() + self._client.sip_call_transfer(settings, completion=completion_callback(future)) + await future + + async def sip_refer(self, settings): + future = self._loop.create_future() + self._client.sip_refer(settings, completion=completion_callback(future)) + await future + async def start_recording(self, streaming_settings, stream_id, force_new): future = self._loop.create_future() self._client.start_recording( @@ -503,6 +518,16 @@ class DailyTransportClient(EventHandler): self._client.stop_recording(stream_id, completion=completion_callback(future)) await future + async def send_prebuilt_chat_message(self, message: str, user_name: str | None = None): + if not self._joined: + return + + future = self._loop.create_future() + self._client.send_prebuilt_chat_message( + message, user_name=user_name, completion=completion_callback(future) + ) + await future + async def capture_participant_transcription(self, participant_id: str): if not self._params.transcription_enabled: return @@ -955,12 +980,30 @@ class DailyTransport(BaseTransport): async def stop_dialout(self, participant_id): await self._client.stop_dialout(participant_id) + async def send_dtmf(self, settings): + await self._client.send_dtmf(settings) + + async def sip_call_transfer(self, settings): + await self._client.sip_call_transfer(settings) + + async def sip_refer(self, settings): + await self._client.sip_refer(settings) + async def start_recording(self, streaming_settings=None, stream_id=None, force_new=None): await self._client.start_recording(streaming_settings, stream_id, force_new) async def stop_recording(self, stream_id=None): await self._client.stop_recording(stream_id) + async def send_prebuilt_chat_message(self, message: str, user_name: str | None = None): + """Sends a chat message to Daily's Prebuilt main room. + + Args: + message: The chat message to send + user_name: Optional user name that will appear as sender of the message + """ + await self._client.send_prebuilt_chat_message(message, user_name) + async def capture_participant_transcription(self, participant_id: str): await self._client.capture_participant_transcription(participant_id) diff --git a/src/pipecat/transports/services/helpers/daily_rest.py b/src/pipecat/transports/services/helpers/daily_rest.py index 4f15fc28a..c17af897c 100644 --- a/src/pipecat/transports/services/helpers/daily_rest.py +++ b/src/pipecat/transports/services/helpers/daily_rest.py @@ -1,26 +1,32 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # -""" -Daily REST Helpers +"""Daily REST Helpers. Methods that wrap the Daily API to create rooms, check room URLs, and get meeting tokens. - """ -import aiohttp import time - +from typing import Literal, Optional from urllib.parse import urlparse -from pydantic import Field, BaseModel, ValidationError -from typing import Literal, Optional +import aiohttp +from pydantic import BaseModel, Field, ValidationError class DailyRoomSipParams(BaseModel): + """SIP configuration parameters for Daily rooms. + + Attributes: + display_name: Name shown for the SIP endpoint + video: Whether video is enabled for SIP + sip_mode: SIP connection mode, typically 'dial-in' + num_endpoints: Number of allowed SIP endpoints + """ + display_name: str = "sw-sip-dialin" video: bool = False sip_mode: str = "dial-in" @@ -28,7 +34,19 @@ class DailyRoomSipParams(BaseModel): class DailyRoomProperties(BaseModel, extra="allow"): - exp: float = Field(default_factory=lambda: time.time() + 5 * 60) + """Properties for configuring a Daily room. + + Attributes: + exp: Optional Unix epoch timestamp for room expiration (e.g., time.time() + 300 for 5 minutes) + enable_chat: Whether chat is enabled in the room + enable_emoji_reactions: Whether emoji reactions are enabled + eject_at_room_exp: Whether to remove participants when room expires + enable_dialout: Whether SIP dial-out is enabled + sip: SIP configuration parameters + sip_uri: SIP URI information returned by Daily + """ + + exp: Optional[float] = None enable_chat: bool = False enable_emoji_reactions: bool = False eject_at_room_exp: bool = True @@ -38,6 +56,11 @@ class DailyRoomProperties(BaseModel, extra="allow"): @property def sip_endpoint(self) -> str: + """Get the SIP endpoint URI if available. + + Returns: + str: SIP endpoint URI or empty string if not available + """ if not self.sip_uri: return "" else: @@ -45,12 +68,32 @@ class DailyRoomProperties(BaseModel, extra="allow"): class DailyRoomParams(BaseModel): + """Parameters for creating a Daily room. + + Attributes: + name: Optional custom name for the room + privacy: Room privacy setting ('private' or 'public') + properties: Room configuration properties + """ + name: Optional[str] = None privacy: Literal["private", "public"] = "public" properties: DailyRoomProperties = Field(default_factory=DailyRoomProperties) class DailyRoomObject(BaseModel): + """Represents a Daily room returned by the API. + + Attributes: + id: Unique room identifier + name: Room name + api_created: Whether room was created via API + privacy: Room privacy setting ('private' or 'public') + url: Full URL for joining the room + created_at: Timestamp of room creation in ISO 8601 format (e.g., "2019-01-26T09:01:22.000Z"). + config: Room configuration properties + """ + id: str name: str api_created: bool @@ -61,6 +104,16 @@ class DailyRoomObject(BaseModel): class DailyRESTHelper: + """Helper class for interacting with Daily's REST API. + + Provides methods for creating, managing, and accessing Daily rooms. + + Args: + daily_api_key: Your Daily API key + daily_api_url: Daily API base URL (e.g. "https://api.daily.co/v1") + aiohttp_session: Async HTTP session for making requests + """ + def __init__( self, *, @@ -73,13 +126,40 @@ class DailyRESTHelper: self.aiohttp_session = aiohttp_session def get_name_from_url(self, room_url: str) -> str: + """Extract room name from a Daily room URL. + + Args: + room_url: Full Daily room URL + + Returns: + str: Room name portion of the URL + """ return urlparse(room_url).path[1:] async def get_room_from_url(self, room_url: str) -> DailyRoomObject: + """Get room details from a Daily room URL. + + Args: + room_url: Full Daily room URL + + Returns: + DailyRoomObject: DailyRoomObject instance for the room + """ room_name = self.get_name_from_url(room_url) return await self._get_room_from_name(room_name) async def create_room(self, params: DailyRoomParams) -> DailyRoomObject: + """Create a new Daily room. + + Args: + params: Room configuration parameters + + Returns: + DailyRoomObject: DailyRoomObject instance for the created room + + Raises: + Exception: If room creation fails or response is invalid + """ headers = {"Authorization": f"Bearer {self.daily_api_key}"} json = {**params.model_dump(exclude_none=True)} async with self.aiohttp_session.post( @@ -101,6 +181,19 @@ class DailyRESTHelper: async def get_token( self, room_url: str, expiry_time: float = 60 * 60, owner: bool = True ) -> str: + """Generate a meeting token for user to join a Daily room. + + Args: + room_url: Daily room URL + expiry_time: Token validity duration in seconds (default: 1 hour) + owner: Whether token has owner privileges + + Returns: + str: Meeting token + + Raises: + Exception: If token generation fails or room URL is missing + """ if not room_url: raise Exception( "No Daily room specified. You must specify a Daily room in order a token to be generated." @@ -124,10 +217,29 @@ class DailyRESTHelper: return data["token"] async def delete_room_by_url(self, room_url: str) -> bool: + """Delete a room using its URL. + + Args: + room_url: Daily room URL + + Returns: + bool: True if deletion was successful + """ room_name = self.get_name_from_url(room_url) return await self.delete_room_by_name(room_name) async def delete_room_by_name(self, room_name: str) -> bool: + """Delete a room using its name. + + Args: + room_name: Name of the room to delete + + Returns: + bool: True if deletion was successful + + Raises: + Exception: If deletion fails (excluding 404 Not Found) + """ headers = {"Authorization": f"Bearer {self.daily_api_key}"} async with self.aiohttp_session.delete( f"{self.daily_api_url}/rooms/{room_name}", headers=headers @@ -139,6 +251,17 @@ class DailyRESTHelper: return True async def _get_room_from_name(self, room_name: str) -> DailyRoomObject: + """Internal method to get room details by name. + + Args: + room_name: Name of the room + + Returns: + DailyRoomObject: DailyRoomObject instance for the room + + Raises: + Exception: If room is not found or response is invalid + """ headers = {"Authorization": f"Bearer {self.daily_api_key}"} async with self.aiohttp_session.get( f"{self.daily_api_url}/rooms/{room_name}", headers=headers diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index f53c8332f..0cf1d16dc 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -8,6 +8,7 @@ import asyncio from dataclasses import dataclass from typing import Any, Awaitable, Callable, List +from loguru import logger from pydantic import BaseModel from pipecat.audio.utils import resample_audio @@ -28,8 +29,6 @@ from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: from livekit import rtc from tenacity import retry, stop_after_attempt, wait_exponential @@ -324,22 +323,13 @@ class LiveKitInputTransport(BaseInputTransport): logger.info("LiveKitInputTransport started") async def stop(self, frame: EndFrame): - if self._audio_in_task: - self._audio_in_task.cancel() - try: - await self._audio_in_task - except asyncio.CancelledError: - pass await super().stop(frame) await self._client.disconnect() + if self._audio_in_task: + self._audio_in_task.cancel() + await self._audio_in_task logger.info("LiveKitInputTransport stopped") - async def process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, EndFrame): - await self.stop(frame) - else: - await super().process_frame(frame, direction) - async def cancel(self, frame: CancelFrame): await super().cancel(frame) await self._client.disconnect() @@ -411,12 +401,6 @@ class LiveKitOutputTransport(BaseOutputTransport): await self._client.disconnect() logger.info("LiveKitOutputTransport stopped") - async def process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, EndFrame): - await self.stop(frame) - else: - await super().process_frame(frame, direction) - async def cancel(self, frame: CancelFrame): await super().cancel(frame) await self._client.disconnect() @@ -526,12 +510,6 @@ class LiveKitTransport(BaseTransport): async def _on_disconnected(self): await self._call_event_handler("on_disconnected") - # Attempt to reconnect - try: - await self._client.connect() - await self._call_event_handler("on_connected") - except Exception as e: - logger.error(f"Failed to reconnect: {e}") async def _on_participant_connected(self, participant_id: str): await self._call_event_handler("on_participant_connected", participant_id) diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 936764345..9c4cab12c 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/utils/test_frame_processor.py b/src/pipecat/utils/test_frame_processor.py index e46bae7ad..fde476007 100644 --- a/src/pipecat/utils/test_frame_processor.py +++ b/src/pipecat/utils/test_frame_processor.py @@ -1,4 +1,5 @@ from typing import List + from pipecat.processors.frame_processor import FrameProcessor diff --git a/src/pipecat/utils/text/base_text_filter.py b/src/pipecat/utils/text/base_text_filter.py index 4e814d7f1..bcc370874 100644 --- a/src/pipecat/utils/text/base_text_filter.py +++ b/src/pipecat/utils/text/base_text_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index 2344d3779..6ec705d69 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -117,8 +117,7 @@ class MarkdownTextFilter(BaseTextFilter): # def _remove_code_blocks(self, text: str) -> str: - """ - Main method to remove code blocks from the input text. + """Main method to remove code blocks from the input text. Handles interruptions and delegates to specific methods based on the current state. """ if self._interrupted: @@ -135,8 +134,7 @@ class MarkdownTextFilter(BaseTextFilter): return self._handle_not_in_code_block(match, text, code_block_pattern) def _handle_in_code_block(self, match, text): - """ - Handle text when we're currently inside a code block. + """Handle text when we're currently inside a code block. If we find the end of the block, return text after it. Otherwise, skip the content. """ if match: @@ -146,8 +144,7 @@ class MarkdownTextFilter(BaseTextFilter): return "" # Skip content inside code block def _handle_not_in_code_block(self, match, text, code_block_pattern): - """ - Handle text when we're not currently inside a code block. + """Handle text when we're not currently inside a code block. Delegate to specific methods based on whether we find a code block delimiter. """ if not match: @@ -159,16 +156,14 @@ class MarkdownTextFilter(BaseTextFilter): return self._handle_code_block_within_text(text, code_block_pattern) def _handle_start_of_code_block(self, text, start_index): - """ - Handle the case where we find the start of a code block. + """Handle the case where we find the start of a code block. Return any text before the code block and set the state to inside a code block. """ self._in_code_block = True return text[:start_index].strip() def _handle_code_block_within_text(self, text, code_block_pattern): - """ - Handle the case where we find a code block within the text. + """Handle the case where we find a code block within the text. If it's a complete code block, remove it and return surrounding text. If it's the start of a code block, return text before it and set state. """ @@ -182,8 +177,7 @@ class MarkdownTextFilter(BaseTextFilter): # Filter tables # def remove_tables(self, text: str) -> str: - """ - Remove tables from the input text, handling cases where + """Remove tables from the input text, handling cases where both start and end tags are in the same input. """ if self._interrupted: diff --git a/src/pipecat/utils/time.py b/src/pipecat/utils/time.py index 0f6ca1076..154d1f0a7 100644 --- a/src/pipecat/utils/time.py +++ b/src/pipecat/utils/time.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 14f1b541a..aa9f1d971 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -11,8 +11,7 @@ _ID = itertools.count() def obj_id() -> int: - """ - Generate a unique id for an object. + """Generate a unique id for an object. >>> obj_id() 0 diff --git a/src/pipecat/vad/silero.py b/src/pipecat/vad/silero.py index 7ec938dbd..e78826dfe 100644 --- a/src/pipecat/vad/silero.py +++ b/src/pipecat/vad/silero.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/vad/vad_analyzer.py b/src/pipecat/vad/vad_analyzer.py index b29b10ef9..14cb5d059 100644 --- a/src/pipecat/vad/vad_analyzer.py +++ b/src/pipecat/vad/vad_analyzer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/tests/integration/integration_azure_llm.py b/tests/integration/integration_azure_llm.py index 5a2b68c37..8e49e9d04 100644 --- a/tests/integration/integration_azure_llm.py +++ b/tests/integration/integration_azure_llm.py @@ -1,17 +1,17 @@ -import unittest - import asyncio import os +import unittest + +from openai.types.chat import ( + ChatCompletionSystemMessageParam, +) + from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, ) from pipecat.services.azure import AzureLLMService -from openai.types.chat import ( - ChatCompletionSystemMessageParam, -) - if __name__ == "__main__": @unittest.skip("Skip azure integration test") diff --git a/tests/integration/integration_ollama_llm.py b/tests/integration/integration_ollama_llm.py index ced24ed68..085500cb8 100644 --- a/tests/integration/integration_ollama_llm.py +++ b/tests/integration/integration_ollama_llm.py @@ -1,14 +1,14 @@ -import unittest - import asyncio -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +import unittest from openai.types.chat import ( ChatCompletionSystemMessageParam, ) + +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.services.ollama import OLLamaLLMService if __name__ == "__main__": diff --git a/tests/integration/integration_openai_llm.py b/tests/integration/integration_openai_llm.py index 164dcba8d..c788936a1 100644 --- a/tests/integration/integration_openai_llm.py +++ b/tests/integration/integration_openai_llm.py @@ -3,17 +3,16 @@ import json import os from typing import List -from pipecat.services.openai import OpenAILLMContextFrame, OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.frames.frames import LLMFullResponseStartFrame, LLMFullResponseEndFrame, TextFrame -from pipecat.utils.test_frame_processor import TestFrameProcessor from openai.types.chat import ( ChatCompletionSystemMessageParam, ChatCompletionToolParam, ChatCompletionUserMessageParam, ) -from pipecat.services.openai import OpenAILLMService +from pipecat.frames.frames import LLMFullResponseEndFrame, LLMFullResponseStartFrame, TextFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService +from pipecat.utils.test_frame_processor import TestFrameProcessor tools = [ ChatCompletionToolParam( diff --git a/tests/test_aggregators.py b/tests/test_aggregators.py index 76834183c..dcf27ad6e 100644 --- a/tests/test_aggregators.py +++ b/tests/test_aggregators.py @@ -3,23 +3,20 @@ import doctest import functools import unittest -from pipecat.processors.aggregators.gated import GatedAggregator -from pipecat.processors.aggregators.sentence import SentenceAggregator -from pipecat.processors.text_transformer import StatelessTextTransformer - -from pipecat.pipeline.parallel_pipeline import ParallelPipeline - from pipecat.frames.frames import ( AudioRawFrame, EndFrame, + Frame, ImageRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, - Frame, TextFrame, ) - +from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators.gated import GatedAggregator +from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.text_transformer import StatelessTextTransformer class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase): diff --git a/tests/test_ai_services.py b/tests/test_ai_services.py index 975f7e20c..13aa20467 100644 --- a/tests/test_ai_services.py +++ b/tests/test_ai_services.py @@ -1,9 +1,8 @@ import unittest - from typing import AsyncGenerator -from pipecat.services.ai_services import AIService, match_endofsentence from pipecat.frames.frames import EndFrame, Frame, TextFrame +from pipecat.services.ai_services import AIService, match_endofsentence class SimpleAIService(AIService): diff --git a/tests/test_langchain.py b/tests/test_langchain.py index d30d213bd..97c97f133 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -1,11 +1,14 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import unittest +from langchain.prompts import ChatPromptTemplate +from langchain_core.language_models import FakeStreamingListLLM + from pipecat.frames.frames import ( EndFrame, LLMFullResponseEndFrame, @@ -25,9 +28,6 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.frame_processor import FrameProcessor from pipecat.processors.frameworks.langchain import LangchainProcessor -from langchain.prompts import ChatPromptTemplate -from langchain_core.language_models import FakeStreamingListLLM - class TestLangchain(unittest.IsolatedAsyncioTestCase): class MockProcessor(FrameProcessor): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ba82974bc..7d703d5ed 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,12 +2,11 @@ import asyncio import unittest from unittest.mock import Mock -from pipecat.processors.aggregators.sentence import SentenceAggregator -from pipecat.processors.text_transformer import StatelessTextTransformer -from pipecat.processors.frame_processor import FrameProcessor from pipecat.frames.frames import EndFrame, TextFrame - from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.processors.text_transformer import StatelessTextTransformer class TestDailyPipeline(unittest.IsolatedAsyncioTestCase):