From 414dcf9810a91e163b679cd5996cc1d818549293 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 11:06:09 -0500 Subject: [PATCH 01/90] Improve TOC in sidebar, fix missing services --- .github/workflows/generate_docs.yaml | 47 ----------- docs/api/conf.py | 112 +++++++++++++++++++++++++-- docs/api/generate_docs.py | 104 ------------------------- docs/api/index.rst | 56 ++++++-------- docs/api/requirements.txt | 2 +- src/pipecat/services/assemblyai.py | 18 +++-- src/pipecat/services/rime.py | 6 ++ 7 files changed, 146 insertions(+), 199 deletions(-) delete mode 100644 .github/workflows/generate_docs.yaml delete mode 100644 docs/api/generate_docs.py diff --git a/.github/workflows/generate_docs.yaml b/.github/workflows/generate_docs.yaml deleted file mode 100644 index 242984d8a..000000000 --- a/.github/workflows/generate_docs.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Generate API Documentation - -on: - release: - types: [published] # Run on new release - workflow_dispatch: # Manual trigger - -jobs: - update-docs: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r docs/api/requirements.txt - pip install . - - - name: Generate API documentation - run: | - cd docs/api - python generate_docs.py - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 - with: - commit-message: 'docs: Update API documentation' - title: 'docs: Update API documentation' - body: | - Automated PR to update API documentation. - - - Generated using `docs/api/generate_docs.py` - - Triggered by: ${{ github.event_name }} - branch: update-api-docs - delete-branch: true - labels: | - documentation diff --git a/docs/api/conf.py b/docs/api/conf.py index fffaf90b2..c046092e2 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -1,5 +1,12 @@ +import importlib +import logging import sys from pathlib import Path +from typing import Dict, List, Optional + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger("sphinx-build") # Add source directory to path docs_dir = Path(__file__).parent @@ -17,6 +24,7 @@ extensions = [ "sphinx.ext.napoleon", "sphinx.ext.viewcode", "sphinx.ext.intersphinx", + "sphinx.ext.coverage", ] # Napoleon settings @@ -32,13 +40,72 @@ autodoc_default_options = { "undoc-members": True, "exclude-members": "__weakref__", "no-index": True, + "show-inheritance": True, } # HTML output settings html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] autodoc_typehints = "description" -html_show_sphinx = False # Remove "Built with Sphinx" +html_show_sphinx = False + + +def get_installed_services() -> Dict[str, Optional[str]]: + """Scan for installed pipecat services and return their status. + Returns a dictionary of service names and their import status/error message. + """ + services_dir = project_root / "src" / "pipecat" / "services" + services_status = {} + + if not services_dir.exists(): + logger.warning(f"Services directory not found: {services_dir}") + return services_status + + for item in services_dir.iterdir(): + if item.is_dir() and not item.name.startswith("_") and not item.name == "to_be_updated": + service_name = item.name + try: + module = importlib.import_module(f"pipecat.services.{service_name}") + services_status[service_name] = None # None indicates success + logger.info(f"Found service: {service_name} at {module.__file__}") + except ImportError as e: + services_status[service_name] = str(e) + logger.warning(f"Failed to import {service_name}: {e}") + + return services_status + + +def generate_services_rst() -> str: + """Generate RST content for services section.""" + services = get_installed_services() + + # Sort services into successful and failed imports + successful = [name for name, status in services.items() if status is None] + failed = [(name, status) for name, status in services.items() if status is not None] + + rst_content = [ + "Services", + "~~~~~~~~", + "", + "Successfully Detected Services:", + "", + ] + + for service in sorted(successful): + rst_content.append(f"* :mod:`pipecat.services.{service}`") + + if failed: + rst_content.extend( + [ + "", + "Services with Import Issues:", + "", + ] + ) + for service, error in sorted(failed): + rst_content.append(f"* {service} (Import failed: {error})") + + return "\n".join(rst_content) def setup(app): @@ -55,12 +122,21 @@ def setup(app): import shutil shutil.rmtree(output_dir) + logger.info(f"Cleaned existing documentation in {output_dir}") - print(f"Generating API documentation...") - print(f"Output directory: {output_dir}") - print(f"Source directory: {source_dir}") + logger.info(f"Generating API documentation...") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Source directory: {source_dir}") + + # Get installed services + services = get_installed_services() + logger.info(f"Found {len(services)} services") + for service, status in services.items(): + if status is None: + logger.info(f"Service available: {service}") + else: + logger.warning(f"Service import failed: {service} - {status}") - # Similar exclusions as in your generate_docs.py excludes = [ str(project_root / "src/pipecat/processors/gstreamer"), str(project_root / "src/pipecat/transports/network"), @@ -72,7 +148,27 @@ def setup(app): ] try: - main(["-f", "-e", "-M", "--no-toc", "-o", output_dir, source_dir] + excludes) - print("API documentation generated successfully!") + main( + [ + "-f", + "-e", + "-M", + "--no-toc", + "--separate", + "--module-first", + "-o", + output_dir, + source_dir, + ] + + excludes + ) + + logger.info("API documentation generated successfully!") + + # Generate services index file + services_index = Path(output_dir) / "services_index.rst" + services_index.write_text(generate_services_rst()) + logger.info(f"Generated services index at {services_index}") + except Exception as e: - print(f"Error generating API documentation: {e}") + logger.error(f"Error generating API documentation: {e}", exc_info=True) diff --git a/docs/api/generate_docs.py b/docs/api/generate_docs.py deleted file mode 100644 index 972ea3c89..000000000 --- a/docs/api/generate_docs.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 - -import shutil -import subprocess -from pathlib import Path - - -def run_command(command: list[str]) -> None: - """Run a command and exit if it fails.""" - print(f"Running: {' '.join(command)}") - try: - subprocess.run(command, check=True) - except subprocess.CalledProcessError as e: - print(f"Warning: Command failed: {' '.join(command)}") - print(f"Error: {e}") - - -def main(): - docs_dir = Path(__file__).parent - project_root = docs_dir.parent.parent - - # Install documentation requirements - requirements_file = docs_dir / "requirements.txt" - run_command(["pip", "install", "-r", str(requirements_file)]) - - # Install from project root, not docs directory - run_command(["pip", "install", "-e", str(project_root)]) - - # Install all service dependencies - services = [ - "anthropic", - "assemblyai", - "aws", - "azure", - "canonical", - "cartesia", - # "daily", - "deepgram", - "elevenlabs", - "fal", - "fireworks", - "gladia", - "google", - "grok", - "groq", - "langchain", - # "livekit", - "lmnt", - "moondream", - "nim", - "noisereduce", - "openai", - "openpipe", - "playht", - "silero", - "soundfile", - "websocket", - "whisper", - ] - - extras = ",".join(services) - try: - run_command(["pip", "install", "-e", f"{str(project_root)}[{extras}]"]) - except Exception as e: - print(f"Warning: Some dependencies failed to install: {e}") - - # Clean old files - api_dir = docs_dir / "api" - build_dir = docs_dir / "_build" - for dir in [api_dir, build_dir]: - if dir.exists(): - shutil.rmtree(dir) - - # Generate API documentation - run_command( - [ - "sphinx-apidoc", - "-f", # Force overwrite - "-e", # Put each module on its own page - "-M", # Put module documentation before submodule - "--no-toc", # Don't generate modules.rst (cleaner structure) - "-o", - str(api_dir), # Output directory - str(project_root / "src/pipecat"), - # Exclude problematic files and directories - str(project_root / "src/pipecat/processors/gstreamer"), # Optional gstreamer - str(project_root / "src/pipecat/transports/network"), # Pydantic issues - str(project_root / "src/pipecat/transports/services"), # Pydantic issues - str(project_root / "src/pipecat/transports/local"), # Optional dependencies - str(project_root / "src/pipecat/services/to_be_updated"), # Exclude to_be_updated - "**/test_*.py", # Test files - "**/tests/*.py", # Test files - ] - ) - - # Build HTML documentation - run_command(["sphinx-build", "-b", "html", str(docs_dir), str(build_dir / "html")]) - - print("\nDocumentation generated successfully!") - print(f"HTML docs: {build_dir}/html/index.html") - - -if __name__ == "__main__": - main() diff --git a/docs/api/index.rst b/docs/api/index.rst index 68afcb980..a878c0c10 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -13,61 +13,55 @@ Quick Links * `GitHub Repository `_ * `Website `_ - API Reference ------------- Core Components ~~~~~~~~~~~~~~~ -* :mod:`pipecat.frames` -* :mod:`pipecat.processors` -* :mod:`pipecat.pipeline` +* :mod:`Frames ` +* :mod:`Processors ` +* :mod:`Pipeline ` Audio Processing ~~~~~~~~~~~~~~~~ -* :mod:`pipecat.audio` -* :mod:`pipecat.vad` - -Services -~~~~~~~~ - -* :mod:`pipecat.services` +* :mod:`Audio ` +* :mod:`VAD ` Transport & Serialization ~~~~~~~~~~~~~~~~~~~~~~~~~ -* :mod:`pipecat.transports` -* :mod:`pipecat.serializers` +* :mod:`Transports ` +* :mod:`Serializers ` Utilities ~~~~~~~~~ -* :mod:`pipecat.clocks` -* :mod:`pipecat.metrics` -* :mod:`pipecat.sync` -* :mod:`pipecat.transcriptions` -* :mod:`pipecat.utils` +* :mod:`Clocks ` +* :mod:`Metrics ` +* :mod:`Sync ` +* :mod:`Transcriptions ` +* :mod:`Utils ` .. toctree:: :maxdepth: 2 :caption: API Reference :hidden: - api/pipecat.audio - api/pipecat.clocks - api/pipecat.frames - api/pipecat.metrics - api/pipecat.pipeline - api/pipecat.processors - api/pipecat.serializers - api/pipecat.services - api/pipecat.sync - api/pipecat.transcriptions - api/pipecat.transports - api/pipecat.utils - api/pipecat.vad + Audio + Clocks + Frames + Metrics + Pipeline + Processors + Serializers + Services + Sync + Transcriptions + Transports + Utils + VAD Indices and tables ================== diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt index f571b6ce0..d57bcd00f 100644 --- a/docs/api/requirements.txt +++ b/docs/api/requirements.txt @@ -3,4 +3,4 @@ sphinx-rtd-theme sphinx-markdown-builder sphinx-autodoc-typehints toml -pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,silero,soundfile,websocket,whisper] +pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,riva,silero,simli,soundfile,websocket,whisper] \ No newline at end of file diff --git a/src/pipecat/services/assemblyai.py b/src/pipecat/services/assemblyai.py index a96589e5a..36a7e92ff 100644 --- a/src/pipecat/services/assemblyai.py +++ b/src/pipecat/services/assemblyai.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import asyncio from typing import AsyncGenerator @@ -67,8 +73,7 @@ class AssemblyAISTTService(STTService): await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """ - Process an audio chunk for STT transcription. + """Process an audio chunk for STT transcription. This method streams the audio data to AssemblyAI for real-time transcription. Transcription results are handled asynchronously via callback functions. @@ -83,8 +88,7 @@ class AssemblyAISTTService(STTService): yield None async def _connect(self): - """ - Establish a connection to the AssemblyAI real-time transcription service. + """Establish a connection to the AssemblyAI real-time transcription service. This method sets up the necessary callback functions and initializes the AssemblyAI transcriber. @@ -95,8 +99,7 @@ class AssemblyAISTTService(STTService): logger.info(f"{self}: Connected to AssemblyAI") def on_data(transcript: aai.RealtimeTranscript): - """ - Callback for handling incoming transcription data. + """Callback for handling incoming transcription data. This function runs in a separate thread from the main asyncio event loop. It creates appropriate transcription frames and schedules them to be @@ -121,8 +124,7 @@ class AssemblyAISTTService(STTService): asyncio.run_coroutine_threadsafe(self.push_frame(frame), self._loop) def on_error(error: aai.RealtimeError): - """ - Callback for handling errors from AssemblyAI. + """Callback for handling errors from AssemblyAI. Like on_data, this runs in a separate thread and schedules error handling in the main event loop. diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 6cf2c8129..79614cb76 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + from typing import AsyncGenerator, Optional import aiohttp From e14399727bbae871fd7057abb8487ae8c16a5b0c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 11:06:53 -0500 Subject: [PATCH 02/90] Add README and build script for local testing --- docs/api/README.md | 78 ++++++++++++++++++++++++++++++++++++++++++ docs/api/build-docs.sh | 10 ++++++ 2 files changed, 88 insertions(+) create mode 100644 docs/api/README.md create mode 100755 docs/api/build-docs.sh diff --git a/docs/api/README.md b/docs/api/README.md new file mode 100644 index 000000000..c0943653f --- /dev/null +++ b/docs/api/README.md @@ -0,0 +1,78 @@ +# Pipecat Documentation + +This directory contains the source files for auto-generating Pipecat's server API reference documentation. + +## Setup + +1. Install documentation dependencies: + +```bash +pip install -r requirements.txt +``` + +2. Make the build script executable: + +```bash +chmod +x build-docs.sh +``` + +## Building Documentation + +From this directory, run either: + +```bash +# Using the build script (automatically opens docs when done) +./build-docs.sh + +# Or directly with sphinx-build +sphinx-build -b html . _build/html -W --keep-going +``` + +## Viewing Documentation + +The built documentation will be available at `_build/html/index.html`. To open: + +```bash +# On MacOS +open _build/html/index.html + +# On Linux +xdg-open _build/html/index.html + +# On Windows +start _build/html/index.html +``` + +## Directory Structure + +``` +. +├── api/ # Auto-generated API documentation +├── _build/ # Built documentation +├── _static/ # Static files (images, css, etc.) +├── conf.py # Sphinx configuration +├── index.rst # Main documentation entry point +├── requirements.txt # Documentation dependencies +└── build-docs.sh # Build script matching ReadTheDocs configuration +``` + +## Notes + +- Documentation is auto-generated from Python docstrings +- Service modules are automatically detected and included +- The build process matches our ReadTheDocs configuration +- Warnings are treated as errors (-W flag) to maintain consistency +- The --keep-going flag ensures all errors are reported + +## Troubleshooting + +If you encounter missing service modules: + +1. Verify the service is installed with its extras: `pip install pipecat-ai[service-name]` +2. Check the build logs for import errors +3. Ensure the service module is properly initialized in the package + +For more information: + +- [ReadTheDocs Configuration](.readthedocs.yaml) +- [Sphinx Documentation](https://www.sphinx-doc.org/) diff --git a/docs/api/build-docs.sh b/docs/api/build-docs.sh new file mode 100755 index 000000000..46d025404 --- /dev/null +++ b/docs/api/build-docs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +# Clean previous build +rm -rf _build + +# Build docs matching ReadTheDocs configuration +sphinx-build -b html -d _build/doctrees . _build/html -W --keep-going + +# Open docs (MacOS) +open _build/html/index.html \ No newline at end of file From f3ed12c30b0a93407fafb0c62f36117a7ab81c6d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 11:11:53 -0500 Subject: [PATCH 03/90] Clean up module and package display names --- docs/api/conf.py | 98 ++++++++++++++---------------------------------- 1 file changed, 29 insertions(+), 69 deletions(-) diff --git a/docs/api/conf.py b/docs/api/conf.py index c046092e2..d4203f16e 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -1,8 +1,6 @@ -import importlib import logging import sys from pathlib import Path -from typing import Dict, List, Optional # Configure logging logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") @@ -24,7 +22,6 @@ extensions = [ "sphinx.ext.napoleon", "sphinx.ext.viewcode", "sphinx.ext.intersphinx", - "sphinx.ext.coverage", ] # Napoleon settings @@ -50,62 +47,26 @@ autodoc_typehints = "description" html_show_sphinx = False -def get_installed_services() -> Dict[str, Optional[str]]: - """Scan for installed pipecat services and return their status. - Returns a dictionary of service names and their import status/error message. - """ - services_dir = project_root / "src" / "pipecat" / "services" - services_status = {} +def clean_title(title: str) -> str: + """Automatically clean module titles.""" + # Remove everything after space (like 'module', 'processor', etc.) + title = title.split(" ")[0] - if not services_dir.exists(): - logger.warning(f"Services directory not found: {services_dir}") - return services_status + # Get the last part of the dot-separated path + parts = title.split(".") + title = parts[-1] - for item in services_dir.iterdir(): - if item.is_dir() and not item.name.startswith("_") and not item.name == "to_be_updated": - service_name = item.name - try: - module = importlib.import_module(f"pipecat.services.{service_name}") - services_status[service_name] = None # None indicates success - logger.info(f"Found service: {service_name} at {module.__file__}") - except ImportError as e: - services_status[service_name] = str(e) - logger.warning(f"Failed to import {service_name}: {e}") + # Handle special cases for common acronyms + acronyms = ["ai", "aws", "api", "vad"] + words = title.split("_") + cleaned_words = [] + for word in words: + if word.lower() in acronyms: + cleaned_words.append(word.upper()) + else: + cleaned_words.append(word.capitalize()) - return services_status - - -def generate_services_rst() -> str: - """Generate RST content for services section.""" - services = get_installed_services() - - # Sort services into successful and failed imports - successful = [name for name, status in services.items() if status is None] - failed = [(name, status) for name, status in services.items() if status is not None] - - rst_content = [ - "Services", - "~~~~~~~~", - "", - "Successfully Detected Services:", - "", - ] - - for service in sorted(successful): - rst_content.append(f"* :mod:`pipecat.services.{service}`") - - if failed: - rst_content.extend( - [ - "", - "Services with Import Issues:", - "", - ] - ) - for service, error in sorted(failed): - rst_content.append(f"* {service} (Import failed: {error})") - - return "\n".join(rst_content) + return " ".join(cleaned_words) def setup(app): @@ -128,15 +89,6 @@ def setup(app): logger.info(f"Output directory: {output_dir}") logger.info(f"Source directory: {source_dir}") - # Get installed services - services = get_installed_services() - logger.info(f"Found {len(services)} services") - for service, status in services.items(): - if status is None: - logger.info(f"Service available: {service}") - else: - logger.warning(f"Service import failed: {service} - {status}") - excludes = [ str(project_root / "src/pipecat/processors/gstreamer"), str(project_root / "src/pipecat/transports/network"), @@ -165,10 +117,18 @@ def setup(app): logger.info("API documentation generated successfully!") - # Generate services index file - services_index = Path(output_dir) / "services_index.rst" - services_index.write_text(generate_services_rst()) - logger.info(f"Generated services index at {services_index}") + # Process generated RST files to update titles + for rst_file in Path(output_dir).glob("*.rst"): + content = rst_file.read_text() + lines = content.split("\n") + + # Find and clean up the title + if lines and "=" in lines[1]: # Title is typically the first line + old_title = lines[0] + new_title = clean_title(old_title) + content = content.replace(old_title, new_title) + rst_file.write_text(content) + logger.info(f"Updated title: {old_title} -> {new_title}") except Exception as e: logger.error(f"Error generating API documentation: {e}", exc_info=True) From b5d5a0e9236046f11e1d3516c5db07cb7dbd4624 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 11:15:36 -0500 Subject: [PATCH 04/90] Add special cases for displaying some names --- docs/api/conf.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/docs/api/conf.py b/docs/api/conf.py index d4203f16e..843700feb 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -56,13 +56,32 @@ def clean_title(title: str) -> str: parts = title.split(".") title = parts[-1] - # Handle special cases for common acronyms - acronyms = ["ai", "aws", "api", "vad"] + # Special cases for service names and common acronyms + special_cases = { + "ai": "AI", + "aws": "AWS", + "api": "API", + "vad": "VAD", + "assemblyai": "AssemblyAI", + "deepgram": "Deepgram", + "elevenlabs": "ElevenLabs", + "openai": "OpenAI", + "openpipe": "OpenPipe", + "playht": "PlayHT", + "xtts": "XTTS", + "lmnt": "LMNT", + } + + # Check if the entire title is a special case + if title.lower() in special_cases: + return special_cases[title.lower()] + + # Otherwise, capitalize each word words = title.split("_") cleaned_words = [] for word in words: - if word.lower() in acronyms: - cleaned_words.append(word.upper()) + if word.lower() in special_cases: + cleaned_words.append(special_cases[word.lower()]) else: cleaned_words.append(word.capitalize()) From 276fd86ecb4e4ee898eb29eac331dd4e58b68084 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 13:25:13 -0500 Subject: [PATCH 05/90] More fixes for missing packages --- .readthedocs.yaml | 64 ++++++++++++++++++++++++++++++- docs/api/conf.py | 62 ++++++++++++++++++++++++++++++ docs/api/index.rst | 1 + docs/api/requirements-base.txt | 36 +++++++++++++++++ docs/api/requirements-playht.txt | 3 ++ docs/api/requirements-riva.txt | 3 ++ docs/api/requirements.txt | 6 --- docs/api/rtd-test.sh | 36 +++++++++++++++++ pyproject.toml | 2 +- src/pipecat/services/anthropic.py | 7 ++-- 10 files changed, 208 insertions(+), 12 deletions(-) create mode 100644 docs/api/requirements-base.txt create mode 100644 docs/api/requirements-playht.txt create mode 100644 docs/api/requirements-riva.txt delete mode 100644 docs/api/requirements.txt create mode 100755 docs/api/rtd-test.sh diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 667e789d9..ea03b508e 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,12 +4,74 @@ build: os: ubuntu-22.04 tools: python: '3.12' + jobs: + pre_build: + # Commands to run before the build + - python -m pip install --upgrade pip + - pip install wheel setuptools + post_create_environment: + # Install system dependencies required by some packages + - apt-get update + - apt-get install -y portaudio19-dev python3-pyaudio + post_build: + # Commands to run after the build + - echo "Build completed" sphinx: configuration: docs/api/conf.py + fail_on_warning: false # Set to true if you want builds to fail on warnings python: install: - - requirements: docs/api/requirements.txt + - requirements: docs/api/requirements-base.txt + # Try to install Riva first, fall back to PlayHT if it fails + - requirements: docs/api/requirements-riva.txt || true + - requirements: docs/api/requirements-playht.txt || true - method: pip path: . + extra_requirements: + - anthropic + - assemblyai + - aws + - azure + - canonical + - cartesia + - deepgram + - elevenlabs + - fal + - fireworks + - gladia + - google + - grok + - groq + - krisp + - langchain + - livekit + - lmnt + - moondream + - nim + - noisereduce + - openai + - openpipe + - silero + - simli + - soundfile + - websocket + - whisper + +formats: + - pdf + - epub + - htmlzip + +# Cache dependencies to speed up builds +search: + ranking: + api/*: 5 + getting-started/*: 4 + guides/*: 3 + +# Configure submodules if needed +submodules: + include: all + recursive: true diff --git a/docs/api/conf.py b/docs/api/conf.py index 843700feb..ad969df8c 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -40,6 +40,31 @@ autodoc_default_options = { "show-inheritance": True, } +# Mock imports for optional dependencies +autodoc_mock_imports = [ + "riva", + "livekit", + "pyht", + "anthropic", + "assemblyai", + "boto3", + "azure", + "cartesia", + "deepgram", + "elevenlabs", + "fal", + "gladia", + "google", + "krisp", + "langchain", + "lmnt", + "noisereduce", + "openai", + "openpipe", + "simli", + "soundfile", +] + # HTML output settings html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] @@ -47,6 +72,39 @@ autodoc_typehints = "description" html_show_sphinx = False +def verify_modules(): + """Verify that required modules are available.""" + required_modules = { + "services": [ + "assemblyai", + "aws", + "cartesia", + "deepgram", + "google", + "lmnt", + "riva", + "simli", + ], + "serializers": ["livekit"], + "vad": ["silero", "vad_analyzer"], + } + + missing = [] + for category, modules in required_modules.items(): + for module in modules: + try: + __import__(f"pipecat.{category}.{module}") + logger.info(f"Successfully imported pipecat.{category}.{module}") + except ImportError as e: + missing.append(f"pipecat.{category}.{module}") + logger.warning( + f"Optional module not available: pipecat.{category}.{module} - {str(e)}" + ) + + if missing: + logger.warning(f"Some optional modules are not available: {missing}") + + def clean_title(title: str) -> str: """Automatically clean module titles.""" # Remove everything after space (like 'module', 'processor', etc.) @@ -151,3 +209,7 @@ def setup(app): except Exception as e: logger.error(f"Error generating API documentation: {e}", exc_info=True) + + +# Run module verification +verify_modules() diff --git a/docs/api/index.rst b/docs/api/index.rst index a878c0c10..a8cba911d 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -22,6 +22,7 @@ Core Components * :mod:`Frames ` * :mod:`Processors ` * :mod:`Pipeline ` +* :mod:`Services ` Audio Processing ~~~~~~~~~~~~~~~~ diff --git a/docs/api/requirements-base.txt b/docs/api/requirements-base.txt new file mode 100644 index 000000000..6739a1962 --- /dev/null +++ b/docs/api/requirements-base.txt @@ -0,0 +1,36 @@ +# Sphinx dependencies +sphinx>=8.1.3 +sphinx-rtd-theme +sphinx-markdown-builder +sphinx-autodoc-typehints +toml + +# Install all extras individually to ensure they're properly resolved +pipecat-ai[anthropic] +pipecat-ai[assemblyai] +pipecat-ai[aws] +pipecat-ai[azure] +pipecat-ai[canonical] +pipecat-ai[cartesia] +pipecat-ai[deepgram] +pipecat-ai[elevenlabs] +pipecat-ai[fal] +pipecat-ai[fireworks] +pipecat-ai[gladia] +pipecat-ai[google] +pipecat-ai[grok] +pipecat-ai[groq] +pipecat-ai[krisp] +pipecat-ai[langchain] +pipecat-ai[livekit] +pipecat-ai[lmnt] +pipecat-ai[moondream] +pipecat-ai[nim] +pipecat-ai[noisereduce] +pipecat-ai[openai] +pipecat-ai[openpipe] +pipecat-ai[silero] +pipecat-ai[simli] +pipecat-ai[soundfile] +pipecat-ai[websocket] +pipecat-ai[whisper] \ No newline at end of file diff --git a/docs/api/requirements-playht.txt b/docs/api/requirements-playht.txt new file mode 100644 index 000000000..1f0bc24ea --- /dev/null +++ b/docs/api/requirements-playht.txt @@ -0,0 +1,3 @@ +# Force specific grpcio version for PlayHT +grpcio>=1.68.0 +pipecat-ai[playht] \ No newline at end of file diff --git a/docs/api/requirements-riva.txt b/docs/api/requirements-riva.txt new file mode 100644 index 000000000..6bd4c69f9 --- /dev/null +++ b/docs/api/requirements-riva.txt @@ -0,0 +1,3 @@ +# Force specific grpcio version for Riva +grpcio==1.65.4 +pipecat-ai[riva] \ No newline at end of file diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt deleted file mode 100644 index d57bcd00f..000000000 --- a/docs/api/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -sphinx>=8.1.3 -sphinx-rtd-theme -sphinx-markdown-builder -sphinx-autodoc-typehints -toml -pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,riva,silero,simli,soundfile,websocket,whisper] \ No newline at end of file diff --git a/docs/api/rtd-test.sh b/docs/api/rtd-test.sh new file mode 100755 index 000000000..87d8880b4 --- /dev/null +++ b/docs/api/rtd-test.sh @@ -0,0 +1,36 @@ +#!/bin/bash +set -e + +# Configuration +DOCS_DIR=$(pwd) +PROJECT_ROOT=$(cd ../../ && pwd) +TEST_DIR="/tmp/rtd-test-$(date +%Y%m%d_%H%M%S)" + +echo "Creating test directory: $TEST_DIR" +mkdir -p "$TEST_DIR" +cd "$TEST_DIR" + +# Create single virtual environment +python -m venv venv +source venv/bin/activate + +echo "Installing base dependencies..." +pip install --upgrade pip wheel setuptools +pip install -r "$DOCS_DIR/requirements-base.txt" + +# Try to install optional dependencies, but don't fail if they don't work +echo "Installing Riva dependencies..." +pip install -r "$DOCS_DIR/requirements-riva.txt" || echo "Failed to install Riva dependencies" + +echo "Installing PlayHT dependencies..." +pip install -r "$DOCS_DIR/requirements-playht.txt" || echo "Failed to install PlayHT dependencies" + +echo "Building documentation..." +cd "$DOCS_DIR" +sphinx-build -b html . "_build/html" + +echo "Build complete. Check _build/html directory for output." + +# Print installed packages for verification +echo "Installed packages:" +pip freeze \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index f3f7d6050..f8b122580 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,11 +66,11 @@ openpipe = [ "openpipe~=4.38.0" ] playht = [ "pyht~=0.1.8", "websockets~=13.1" ] riva = [ "nvidia-riva-client~=2.17.0" ] silero = [ "onnxruntime~=1.20.1" ] +simli = [ "simli-ai~=0.1.7"] soundfile = [ "soundfile~=0.12.1" ] together = [ "openai~=1.57.2" ] websocket = [ "websockets~=13.1", "fastapi~=0.115.0" ] whisper = [ "faster-whisper~=1.1.0" ] -simli = [ "simli-ai~=0.1.7"] [tool.setuptools.packages.find] # All the following settings are optional: diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 48102eda7..f0c033375 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -11,7 +11,7 @@ import json import re from asyncio import CancelledError from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union from loguru import logger from PIL import Image @@ -75,8 +75,7 @@ class AnthropicContextAggregatorPair: class AnthropicLLMService(LLMService): - """ - This class implements inference with Anthropic's AI models. + """This class implements inference with Anthropic's AI models. Can provide a custom client via the `client` kwarg, allowing you to use `AsyncAnthropicBedrock` and `AsyncAnthropicVertex` clients @@ -328,7 +327,7 @@ class AnthropicLLMContext(OpenAILLMContext): tools: list[dict] | None = None, tool_choice: dict | None = None, *, - system: str | NotGiven = NOT_GIVEN, + system: Union[str, NotGiven] = NOT_GIVEN, ): super().__init__(messages=messages, tools=tools, tool_choice=tool_choice) From 44c5220104de8b70881b6114a18d4b5f2b3e853c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 13:28:05 -0500 Subject: [PATCH 06/90] Update README --- docs/api/README.md | 41 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/docs/api/README.md b/docs/api/README.md index c0943653f..22b62d45e 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -10,15 +10,17 @@ This directory contains the source files for auto-generating Pipecat's server AP pip install -r requirements.txt ``` -2. Make the build script executable: +2. Make the build scripts executable: ```bash -chmod +x build-docs.sh +chmod +x build-docs.sh rtd-test.py ``` ## Building Documentation -From this directory, run either: +From this directory, you can build the documentation in several ways: + +### Local Build ```bash # Using the build script (automatically opens docs when done) @@ -28,6 +30,24 @@ From this directory, run either: sphinx-build -b html . _build/html -W --keep-going ``` +### ReadTheDocs Test Build + +To test the documentation build process exactly as it would run on ReadTheDocs: + +```bash +./rtd-test.py +``` + +This script: + +- Creates a fresh virtual environment +- Installs all dependencies as specified in requirements files +- Handles conflicting dependencies (like grpcio versions for Riva and PlayHT) +- Builds the documentation in an isolated environment +- Provides detailed logging of the build process + +Use this script to verify your documentation will build correctly on ReadTheDocs before pushing changes. + ## Viewing Documentation The built documentation will be available at `_build/html/index.html`. To open: @@ -52,8 +72,11 @@ start _build/html/index.html ├── _static/ # Static files (images, css, etc.) ├── conf.py # Sphinx configuration ├── index.rst # Main documentation entry point -├── requirements.txt # Documentation dependencies -└── build-docs.sh # Build script matching ReadTheDocs configuration +├── requirements-base.txt # Base documentation dependencies +├── requirements-riva.txt # Riva-specific dependencies +├── requirements-playht.txt # PlayHT-specific dependencies +├── build-docs.sh # Local build script +└── rtd-test.py # ReadTheDocs test build script ``` ## Notes @@ -63,6 +86,7 @@ start _build/html/index.html - The build process matches our ReadTheDocs configuration - Warnings are treated as errors (-W flag) to maintain consistency - The --keep-going flag ensures all errors are reported +- Dependencies are split into multiple requirements files to handle version conflicts ## Troubleshooting @@ -71,6 +95,13 @@ If you encounter missing service modules: 1. Verify the service is installed with its extras: `pip install pipecat-ai[service-name]` 2. Check the build logs for import errors 3. Ensure the service module is properly initialized in the package +4. Run `./rtd-test.py` to test in an isolated environment matching ReadTheDocs + +For dependency conflicts: + +1. Check the requirements files for version specifications +2. Use `rtd-test.py` to verify dependency resolution +3. Consider adding service-specific requirements files if needed For more information: From db7eaed980bc15acea84dfd0b309844f0755c6c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 12 Dec 2024 10:56:02 -0800 Subject: [PATCH 07/90] transport(output): fix non-audio frames sync after audio frames --- CHANGELOG.md | 8 + src/pipecat/transports/base_output.py | 275 +++++++++++--------------- 2 files changed, 121 insertions(+), 162 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac15dffec..5ca41b8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `AWSTTSService` is now deprecated, use `PollyTTSService` instead. +### Fixed + +- Fixed a `BaseOutputTransport` issue that was causing non-audio frames being + processed before the previous audio frames were played. This will allow, for + example, sending a frame `A` after a `TTSSpeakFrame` and the frame `A` will + only be pushed downstream after the audio generated from `TTSSpeakFrame` has + been spoken. + ## [0.0.50] - 2024-12-11 ### Added diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index b25b4c78c..573b67717 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -15,7 +15,6 @@ from PIL import Image from pipecat.audio.vad.vad_analyzer import VAD_STOP_SECS from pipecat.frames.frames import ( - AudioRawFrame, BotSpeakingFrame, BotStartedSpeakingFrame, BotStoppedSpeakingFrame, @@ -52,9 +51,7 @@ class BaseOutputTransport(FrameProcessor): self._sink_clock_task = None # Task to write/send audio and image frames. - self._audio_out_task = None self._camera_out_task = None - self._running_out_tasks = True # These are the images that we should send to the camera at our desired # framerate. @@ -77,22 +74,31 @@ class BaseOutputTransport(FrameProcessor): # Start audio mixer. if self._params.audio_out_mixer: await self._params.audio_out_mixer.start(self._params.audio_out_sample_rate) - self._create_output_tasks() + self._create_camera_task() self._create_sink_tasks() async def stop(self, frame: EndFrame): - # We can't cancel output tasks because there might still be audio - # buffered to be played. - await self._stop_output_tasks() - # Stop audio mixer. - if self._params.audio_out_mixer: - await self._params.audio_out_mixer.stop() + # Let the sink tasks process the queue until they reach this EndFrame. + await self._sink_clock_queue.put((sys.maxsize, frame.id, frame)) + await self._sink_queue.put(frame) + + # At this point we have enqueued an EndFrame and we need to wait for + # that EndFrame to be processed by the sink tasks. We also need to wait + # for these tasks before cancelling the camera and audio tasks below + # because they might be still rendering. + if self._sink_task: + await self._sink_task + if self._sink_clock_task: + await self._sink_clock_task + + # We can now cancel the camera task. + await self._cancel_camera_task() async def cancel(self, frame: CancelFrame): # Since we are cancelling everything it doesn't matter if we cancel sink # tasks first or not. await self._cancel_sink_tasks() - await self._cancel_output_tasks() + await self._cancel_camera_task() async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): pass @@ -103,6 +109,12 @@ class BaseOutputTransport(FrameProcessor): async def write_raw_audio_frames(self, frames: bytes): pass + async def send_audio(self, frame: OutputAudioRawFrame): + await self.queue_frame(frame, FrameDirection.DOWNSTREAM) + + async def send_image(self, frame: OutputImageRawFrame | SpriteFrame): + await self.queue_frame(frame, FrameDirection.DOWNSTREAM) + # # Frame processor # @@ -132,11 +144,8 @@ class BaseOutputTransport(FrameProcessor): await self.push_frame(frame, direction) # Control frames. elif isinstance(frame, EndFrame): - # Process sink tasks. - await self._stop_sink_tasks(frame) - # Now we can stop. await self.stop(frame) - # We finally push EndFrame down so PipelineTask stops nicely. + # Keep pushing EndFrame down so all the pipeline stops nicely. await self.push_frame(frame, direction) elif isinstance(frame, MixerControlFrame) and self._params.audio_out_mixer: await self._params.audio_out_mixer.process_frame(frame) @@ -151,30 +160,16 @@ class BaseOutputTransport(FrameProcessor): else: await self._sink_queue.put(frame) - async def _stop_sink_tasks(self, frame: EndFrame): - # Let the sink tasks process the queue until they reach this EndFrame. - await self._sink_clock_queue.put((sys.maxsize, frame.id, frame)) - await self._sink_queue.put(frame) - - # At this point we have enqueued an EndFrame and we need to wait for - # that EndFrame to be processed by the sink tasks. We also need to wait - # for these tasks before cancelling the camera and audio tasks below - # because they might be still rendering. - if self._sink_task: - await self._sink_task - if self._sink_clock_task: - await self._sink_clock_task - async def _handle_interruptions(self, frame: Frame): if not self.interruptions_allowed: return if isinstance(frame, StartInterruptionFrame): - # Cancel sink and output tasks. + # Cancel sink and camera tasks. await self._cancel_sink_tasks() - await self._cancel_output_tasks() - # Create sink and output tasks. - self._create_output_tasks() + await self._cancel_camera_task() + # Create sink and camera tasks. + self._create_camera_task() self._create_sink_tasks() # Let's send a bot stopped speaking if we have to. await self._bot_stopped_speaking() @@ -183,19 +178,16 @@ class BaseOutputTransport(FrameProcessor): if not self._params.audio_out_enabled: return - if self._params.audio_out_is_live: - await self._audio_out_queue.put(frame) - else: - cls = type(frame) - self._audio_buffer.extend(frame.audio) - while len(self._audio_buffer) >= self._audio_chunk_size: - chunk = cls( - bytes(self._audio_buffer[: self._audio_chunk_size]), - sample_rate=frame.sample_rate, - num_channels=frame.num_channels, - ) - await self._sink_queue.put(chunk) - self._audio_buffer = self._audio_buffer[self._audio_chunk_size :] + cls = type(frame) + self._audio_buffer.extend(frame.audio) + while len(self._audio_buffer) >= self._audio_chunk_size: + chunk = cls( + bytes(self._audio_buffer[: self._audio_chunk_size]), + sample_rate=frame.sample_rate, + num_channels=frame.num_channels, + ) + await self._sink_queue.put(chunk) + self._audio_buffer = self._audio_buffer[self._audio_chunk_size :] async def _handle_image(self, frame: OutputImageRawFrame | SpriteFrame): if not self._params.camera_out_enabled: @@ -244,9 +236,7 @@ class BaseOutputTransport(FrameProcessor): self._sink_clock_task = None async def _sink_frame_handler(self, frame: Frame): - if isinstance(frame, OutputAudioRawFrame): - await self._audio_out_queue.put(frame) - elif isinstance(frame, OutputImageRawFrame): + if isinstance(frame, OutputImageRawFrame): await self._set_camera_image(frame) elif isinstance(frame, SpriteFrame): await self._set_camera_images(frame.images) @@ -256,19 +246,6 @@ class BaseOutputTransport(FrameProcessor): elif not isinstance(frame, EndFrame): await self.push_frame(frame) - async def _sink_task_handler(self): - running = True - while running: - try: - frame = await self._sink_queue.get() - await self._sink_frame_handler(frame) - running = not isinstance(frame, EndFrame) - self._sink_queue.task_done() - except asyncio.CancelledError: - break - except Exception as e: - logger.exception(f"{self} error processing sink queue: {e}") - async def _sink_clock_task_handler(self): running = True while running: @@ -294,48 +271,93 @@ class BaseOutputTransport(FrameProcessor): except Exception as e: logger.exception(f"{self} error processing sink clock queue: {e}") + def _next_frame(self) -> AsyncGenerator[Frame, None]: + async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]: + while True: + try: + frame = await asyncio.wait_for(self._sink_queue.get(), timeout=vad_stop_secs) + yield frame + except asyncio.TimeoutError: + # Notify the bot stopped speaking upstream if necessary. + await self._bot_stopped_speaking() + + async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[Frame, None]: + last_frame_time = 0 + silence = b"\x00" * self._audio_chunk_size + while True: + try: + frame = self._sink_queue.get_nowait() + if isinstance(frame, OutputAudioRawFrame): + frame.audio = await self._params.audio_out_mixer.mix(frame.audio) + last_frame_time = time.time() + yield frame + except asyncio.QueueEmpty: + # Notify the bot stopped speaking upstream if necessary. + diff_time = time.time() - last_frame_time + if diff_time > vad_stop_secs: + await self._bot_stopped_speaking() + # Generate an audio frame with only the mixer's part. + frame = OutputAudioRawFrame( + audio=await self._params.audio_out_mixer.mix(silence), + sample_rate=self._params.audio_out_sample_rate, + num_channels=self._params.audio_out_channels, + ) + yield frame + + vad_stop_secs = ( + self._params.vad_analyzer.params.stop_secs + if self._params.vad_analyzer + else VAD_STOP_SECS + ) + if self._params.audio_out_mixer: + return with_mixer(vad_stop_secs) + else: + return without_mixer(vad_stop_secs) + + async def _sink_task_handler(self): + try: + async for frame in self._next_frame(): + # Notify the bot started speaking upstream if necessary and that + # it's actually speaking. + if isinstance(frame, TTSAudioRawFrame): + await self._bot_started_speaking() + await self.push_frame(BotSpeakingFrame()) + await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) + + # Handle frame. + await self._sink_frame_handler(frame) + + # Also, push frame downstream in case anyone else needs it. + await self.push_frame(frame) + + # Send audio. + if isinstance(frame, OutputAudioRawFrame): + await self.write_raw_audio_frames(frame.audio) + + if isinstance(frame, EndFrame): + break + except asyncio.CancelledError: + pass + except Exception as e: + logger.exception(f"{self} error writing to microphone: {e}") + # - # Output tasks + # Camera task # - def _create_output_tasks(self): + def _create_camera_task(self): loop = self.get_event_loop() # Create camera output queue and task if needed. if self._params.camera_out_enabled: self._camera_out_queue = asyncio.Queue() self._camera_out_task = loop.create_task(self._camera_out_task_handler()) - # Create audio output queue and task if needed. - if self._params.audio_out_enabled: - self._audio_out_queue = asyncio.Queue() - self._audio_out_task = loop.create_task(self._audio_out_task_handler()) - async def _stop_output_tasks(self): - self._running_out_tasks = False - # Stop camera output task. - if self._camera_out_task and self._params.camera_out_enabled: - await self._camera_out_task - # Stop audio output task. - if self._audio_out_task and self._params.audio_out_enabled: - await self._audio_out_task - - async def _cancel_output_tasks(self): + async def _cancel_camera_task(self): # Stop camera output task. if self._camera_out_task and self._params.camera_out_enabled: self._camera_out_task.cancel() await self._camera_out_task self._camera_out_task = None - # Stop audio output task. - if self._audio_out_task and self._params.audio_out_enabled: - self._audio_out_task.cancel() - await self._audio_out_task - self._audio_out_task = None - - # - # Camera out - # - - async def send_image(self, frame: OutputImageRawFrame | SpriteFrame): - await self.queue_frame(frame, FrameDirection.DOWNSTREAM) async def _draw_image(self, frame: OutputImageRawFrame): desired_size = (self._params.camera_out_width, self._params.camera_out_height) @@ -361,7 +383,7 @@ class BaseOutputTransport(FrameProcessor): self._camera_out_frame_index = 0 self._camera_out_frame_duration = 1 / self._params.camera_out_framerate self._camera_out_frame_reset = self._camera_out_frame_duration * 5 - while self._running_out_tasks: + while True: try: if self._params.camera_out_is_live: await self._camera_out_is_live_handler() @@ -400,74 +422,3 @@ class BaseOutputTransport(FrameProcessor): await self._draw_image(image) self._camera_out_queue.task_done() - - # - # Audio out - # - - async def send_audio(self, frame: OutputAudioRawFrame): - await self.queue_frame(frame, FrameDirection.DOWNSTREAM) - - def _next_audio_frame(self) -> AsyncGenerator[AudioRawFrame, None]: - async def without_mixer(vad_stop_secs: float) -> AsyncGenerator[AudioRawFrame, None]: - while self._running_out_tasks or self._bot_speaking: - try: - frame = await asyncio.wait_for( - self._audio_out_queue.get(), timeout=vad_stop_secs - ) - yield frame - except asyncio.TimeoutError: - # Notify the bot stopped speaking upstream if necessary. - await self._bot_stopped_speaking() - - async def with_mixer(vad_stop_secs: float) -> AsyncGenerator[AudioRawFrame, None]: - last_frame_time = 0 - silence = b"\x00" * self._audio_chunk_size - while self._running_out_tasks or self._bot_speaking: - try: - frame = self._audio_out_queue.get_nowait() - frame.audio = await self._params.audio_out_mixer.mix(frame.audio) - last_frame_time = time.time() - yield frame - except asyncio.QueueEmpty: - # Notify the bot stopped speaking upstream if necessary. - diff_time = time.time() - last_frame_time - if diff_time > vad_stop_secs: - await self._bot_stopped_speaking() - # Generate an audio frame with only the mixer's part. - frame = OutputAudioRawFrame( - audio=await self._params.audio_out_mixer.mix(silence), - sample_rate=self._params.audio_out_sample_rate, - num_channels=self._params.audio_out_channels, - ) - yield frame - - vad_stop_secs = ( - self._params.vad_analyzer.params.stop_secs - if self._params.vad_analyzer - else VAD_STOP_SECS - ) - if self._params.audio_out_mixer: - return with_mixer(vad_stop_secs) - else: - return without_mixer(vad_stop_secs) - - async def _audio_out_task_handler(self): - try: - async for frame in self._next_audio_frame(): - # Notify the bot started speaking upstream if necessary and that - # it's actually speaking. - if isinstance(frame, TTSAudioRawFrame): - await self._bot_started_speaking() - await self.push_frame(BotSpeakingFrame()) - await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) - - # Also, push frame downstream in case anyone else needs it. - await self.push_frame(frame) - - # Send audio. - await self.write_raw_audio_frames(frame.audio) - except asyncio.CancelledError: - pass - except Exception as e: - logger.exception(f"{self} error writing to microphone: {e}") From 8631d71d5ac358107d73601a94660da32a526780 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 15:16:37 -0500 Subject: [PATCH 08/90] Fix more missing docs --- .readthedocs.yaml | 8 +--- docs/api/README.md | 2 +- docs/api/conf.py | 68 ++++++++++++++++++++++++---------- docs/api/index.rst | 11 +++++- docs/api/requirements-base.txt | 2 + 5 files changed, 63 insertions(+), 28 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index ea03b508e..e4f64ce91 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,10 +9,6 @@ build: # Commands to run before the build - python -m pip install --upgrade pip - pip install wheel setuptools - post_create_environment: - # Install system dependencies required by some packages - - apt-get update - - apt-get install -y portaudio19-dev python3-pyaudio post_build: # Commands to run after the build - echo "Build completed" @@ -36,6 +32,7 @@ python: - azure - canonical - cartesia + - daily - deepgram - elevenlabs - fal @@ -48,6 +45,7 @@ python: - langchain - livekit - lmnt + - local - moondream - nim - noisereduce @@ -64,14 +62,12 @@ formats: - epub - htmlzip -# Cache dependencies to speed up builds search: ranking: api/*: 5 getting-started/*: 4 guides/*: 3 -# Configure submodules if needed submodules: include: all recursive: true diff --git a/docs/api/README.md b/docs/api/README.md index 22b62d45e..392430071 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -7,7 +7,7 @@ This directory contains the source files for auto-generating Pipecat's server AP 1. Install documentation dependencies: ```bash -pip install -r requirements.txt +pip install -r requirements-base.txt requirements-playht.txt requirements-riva.txt ``` 2. Make the build scripts executable: diff --git a/docs/api/conf.py b/docs/api/conf.py index ad969df8c..2744ded19 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -63,6 +63,16 @@ autodoc_mock_imports = [ "openpipe", "simli", "soundfile", + # Add these new mocks + "pyaudio", + "_tkinter", + "tkinter", + "daily", + "daily_python", + "pydantic.BaseModel", # Mock base pydantic to avoid model conflicts + "pydantic.Field", + "pydantic._internal._model_construction", + "pydantic._internal._fields", ] # HTML output settings @@ -87,19 +97,40 @@ def verify_modules(): ], "serializers": ["livekit"], "vad": ["silero", "vad_analyzer"], + "transports": { + "services": ["daily", "livekit"], + "local": ["audio", "tk"], + "network": ["fastapi_websocket", "websocket_server"], + }, } missing = [] for category, modules in required_modules.items(): - for module in modules: - try: - __import__(f"pipecat.{category}.{module}") - logger.info(f"Successfully imported pipecat.{category}.{module}") - except ImportError as e: - missing.append(f"pipecat.{category}.{module}") - logger.warning( - f"Optional module not available: pipecat.{category}.{module} - {str(e)}" - ) + if isinstance(modules, dict): + # Handle nested structure + for subcategory, submodules in modules.items(): + for module in submodules: + try: + __import__(f"pipecat.{category}.{subcategory}.{module}") + logger.info( + f"Successfully imported pipecat.{category}.{subcategory}.{module}" + ) + except (ImportError, TypeError, NameError) as e: + missing.append(f"pipecat.{category}.{subcategory}.{module}") + logger.warning( + f"Optional module not available: pipecat.{category}.{subcategory}.{module} - {str(e)}" + ) + else: + # Handle flat structure + for module in modules: + try: + __import__(f"pipecat.{category}.{module}") + logger.info(f"Successfully imported pipecat.{category}.{module}") + except (ImportError, TypeError, NameError) as e: + missing.append(f"pipecat.{category}.{module}") + logger.warning( + f"Optional module not available: pipecat.{category}.{module} - {str(e)}" + ) if missing: logger.warning(f"Some optional modules are not available: {missing}") @@ -167,10 +198,8 @@ def setup(app): logger.info(f"Source directory: {source_dir}") excludes = [ + str(project_root / "src/pipecat/pipeline/to_be_updated"), str(project_root / "src/pipecat/processors/gstreamer"), - str(project_root / "src/pipecat/transports/network"), - str(project_root / "src/pipecat/transports/services"), - str(project_root / "src/pipecat/transports/local"), str(project_root / "src/pipecat/services/to_be_updated"), "**/test_*.py", "**/tests/*.py", @@ -179,12 +208,13 @@ def setup(app): try: main( [ - "-f", - "-e", - "-M", - "--no-toc", - "--separate", - "--module-first", + "-f", # Force overwriting + "-e", # Don't generate empty files + "-M", # Put module documentation before submodule documentation + "--no-toc", # Don't create a table of contents file + "--separate", # Put documentation for each module in its own page + "--module-first", # Module documentation before submodule documentation + "--implicit-namespaces", # Added: Handle implicit namespace packages "-o", output_dir, source_dir, @@ -195,7 +225,7 @@ def setup(app): logger.info("API documentation generated successfully!") # Process generated RST files to update titles - for rst_file in Path(output_dir).glob("*.rst"): + for rst_file in Path(output_dir).glob("**/*.rst"): # Changed to recursive glob content = rst_file.read_text() lines = content.split("\n") diff --git a/docs/api/index.rst b/docs/api/index.rst index a8cba911d..f4773eddc 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -22,7 +22,6 @@ Core Components * :mod:`Frames ` * :mod:`Processors ` * :mod:`Pipeline ` -* :mod:`Services ` Audio Processing ~~~~~~~~~~~~~~~~ @@ -30,10 +29,18 @@ Audio Processing * :mod:`Audio ` * :mod:`VAD ` +Services +~~~~~~~~ + +* :mod:`Services ` + Transport & Serialization ~~~~~~~~~~~~~~~~~~~~~~~~~ * :mod:`Transports ` + * :mod:`Local ` + * :mod:`Network ` + * :mod:`Services ` * :mod:`Serializers ` Utilities @@ -46,7 +53,7 @@ Utilities * :mod:`Utils ` .. toctree:: - :maxdepth: 2 + :maxdepth: 3 :caption: API Reference :hidden: diff --git a/docs/api/requirements-base.txt b/docs/api/requirements-base.txt index 6739a1962..ff397d0ef 100644 --- a/docs/api/requirements-base.txt +++ b/docs/api/requirements-base.txt @@ -12,6 +12,7 @@ pipecat-ai[aws] pipecat-ai[azure] pipecat-ai[canonical] pipecat-ai[cartesia] +pipecat-ai[daily] pipecat-ai[deepgram] pipecat-ai[elevenlabs] pipecat-ai[fal] @@ -24,6 +25,7 @@ pipecat-ai[krisp] pipecat-ai[langchain] pipecat-ai[livekit] pipecat-ai[lmnt] +pipecat-ai[local] pipecat-ai[moondream] pipecat-ai[nim] pipecat-ai[noisereduce] From ec082d08888c12217d5addafc3e7b57b60538f85 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 15:32:38 -0500 Subject: [PATCH 09/90] Remove deprecated VAD module --- docs/api/conf.py | 1 + docs/api/index.rst | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/api/conf.py b/docs/api/conf.py index 2744ded19..51e000cbf 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -201,6 +201,7 @@ def setup(app): str(project_root / "src/pipecat/pipeline/to_be_updated"), str(project_root / "src/pipecat/processors/gstreamer"), str(project_root / "src/pipecat/services/to_be_updated"), + str(project_root / "src/pipecat/vad"), # deprecated "**/test_*.py", "**/tests/*.py", ] diff --git a/docs/api/index.rst b/docs/api/index.rst index f4773eddc..ce7c22113 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -27,7 +27,6 @@ Audio Processing ~~~~~~~~~~~~~~~~ * :mod:`Audio ` -* :mod:`VAD ` Services ~~~~~~~~ @@ -69,7 +68,6 @@ Utilities Transcriptions Transports Utils - VAD Indices and tables ================== From 67804edce6eafef06e3bd8d05b3077cccf17fd3a Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 15:39:19 -0500 Subject: [PATCH 10/90] Remove formats from .readthedocs.yaml --- .readthedocs.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index e4f64ce91..379e31465 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -57,11 +57,6 @@ python: - websocket - whisper -formats: - - pdf - - epub - - htmlzip - search: ranking: api/*: 5 From 3c3fd67d96fa54c8785734d0ec9e6604f3da28a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 12 Dec 2024 12:58:45 -0800 Subject: [PATCH 11/90] no longer necessary to call super().process_frame(frame, direction) --- CHANGELOG.md | 6 +++ .../foundational/05-sync-speech-and-image.py | 2 - .../05a-local-sync-speech-and-image.py | 6 --- examples/foundational/06a-image-sync.py | 2 - .../07s-interruptible-google-audio-in.py | 5 -- examples/foundational/09-mirror.py | 2 - examples/foundational/09a-local-mirror.py | 2 - examples/foundational/11-sound-effects.py | 4 -- examples/foundational/12-describe-video.py | 2 - .../12a-describe-video-gemini-flash.py | 2 - .../foundational/12b-describe-video-gpt-4o.py | 2 - .../12c-describe-video-anthropic.py | 2 - .../foundational/13-whisper-transcription.py | 2 - examples/foundational/13a-whisper-local.py | 2 - .../13b-deepgram-transcription.py | 2 - .../foundational/13c-gladia-transcription.py | 2 - .../13d-assemblyai-transcription.py | 2 - .../22b-natural-conversation-proposal.py | 4 -- .../22c-natural-conversation-mixed-llms.py | 41 +++++++------- .../22d-natural-conversation-gemini-audio.py | 6 --- examples/foundational/25-google-audio-in.py | 8 --- examples/moondream-chatbot/bot.py | 8 --- examples/simple-chatbot/server/bot-gemini.py | 2 - examples/simple-chatbot/server/bot-openai.py | 2 - .../storytelling-chatbot/src/processors.py | 4 -- examples/translation-chatbot/bot.py | 4 -- src/pipecat/pipeline/parallel_pipeline.py | 6 --- src/pipecat/pipeline/pipeline.py | 6 --- .../pipeline/sync_parallel_pipeline.py | 6 --- src/pipecat/pipeline/task.py | 4 -- src/pipecat/processors/aggregators/gated.py | 2 - .../aggregators/gated_openai_llm_context.py | 2 - .../processors/aggregators/llm_response.py | 4 -- .../processors/aggregators/sentence.py | 2 - .../processors/aggregators/user_response.py | 2 - .../aggregators/vision_image_frame.py | 2 - src/pipecat/processors/async_generator.py | 2 - .../audio/audio_buffer_processor.py | 2 - src/pipecat/processors/audio/vad/silero.py | 2 - .../processors/filters/frame_filter.py | 2 - .../processors/filters/function_filter.py | 2 - .../processors/filters/identity_filter.py | 1 - .../processors/filters/wake_check_filter.py | 2 - .../filters/wake_notifier_filter.py | 2 - src/pipecat/processors/frame_processor.py | 54 +++++++++++-------- .../processors/frameworks/langchain.py | 2 - src/pipecat/processors/frameworks/rtvi.py | 16 ------ .../processors/gstreamer/pipeline_source.py | 2 - .../processors/idle_frame_processor.py | 2 - src/pipecat/processors/text_transformer.py | 2 - src/pipecat/processors/user_idle_processor.py | 2 - src/pipecat/services/ai_services.py | 2 - src/pipecat/services/simli.py | 1 - src/pipecat/transports/base_input.py | 2 - src/pipecat/transports/base_output.py | 2 - src/pipecat/utils/test_frame_processor.py | 2 - tests/test_langchain.py | 2 - 57 files changed, 56 insertions(+), 212 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ac15dffec..4ab3fc67d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa). +### Changed + +- It's no longer necessary to call `super().process_frame(frame, direction)` if + you subclass and implement `FrameProcessor.process_frame()`. This is all now + done internally and will avoid possible issues if you forget to add it. + ### Deprecated - `AWSTTSService` is now deprecated, use `PollyTTSService` instead. diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 64f85930b..8d5790ac7 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -56,8 +56,6 @@ class MonthPrepender(FrameProcessor): self.prepend_to_next_text_frame = False async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, MonthFrame): self.most_recent_month = frame.month elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame): diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 4a561c073..f6e5f0ce6 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -62,8 +62,6 @@ async def main(): self.text = "" async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): self.text = frame.text await self.push_frame(frame, direction) @@ -75,8 +73,6 @@ async def main(): self.frame = None async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TTSAudioRawFrame): self.audio.extend(frame.audio) self.frame = OutputAudioRawFrame( @@ -90,8 +86,6 @@ async def main(): self.frame = None async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, URLImageRawFrame): self.frame = frame await self.push_frame(frame, direction) diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index eda3c61df..11c894478 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -47,8 +47,6 @@ class ImageSyncAggregator(FrameProcessor): self._waiting_image_bytes = self._waiting_image.tobytes() async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if not isinstance(frame, SystemFrame) and direction == FrameDirection.DOWNSTREAM: await self.push_frame( OutputImageRawFrame( diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 1778e0c62..1db8b0d36 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -82,8 +82,6 @@ class UserAudioCollector(FrameProcessor): self._user_speaking = False async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if isinstance(frame, TranscriptionFrame): # We could gracefully handle both audio input and text/transcription input ... # but let's leave that as an exercise to the reader. :-) @@ -126,7 +124,6 @@ class TranscriptExtractor(FrameProcessor): self._accumulating_transcript = False async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) if isinstance(frame, LLMFullResponseStartFrame): self._processing_llm_response = True self._accumulating_transcript = True @@ -180,8 +177,6 @@ class TanscriptionContextFixup(FrameProcessor): self._context.messages[-1].parts[-1].text += f"\n\n{marker}\n{self._transcript}\n" async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if isinstance(frame, MagicDemoTranscriptionFrame): self._transcript = frame.text elif isinstance(frame, LLMFullResponseEndFrame) or isinstance( diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index a719d54f6..8eaee5750 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -35,8 +35,6 @@ logger.add(sys.stderr, level="DEBUG") class MirrorProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, InputAudioRawFrame): await self.push_frame( OutputAudioRawFrame( diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index 539cca600..4a4a1fee1 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -39,8 +39,6 @@ logger.add(sys.stderr, level="DEBUG") class MirrorProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, InputAudioRawFrame): await self.push_frame( OutputAudioRawFrame( diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index d8692a7f1..50d3f9e33 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -60,8 +60,6 @@ for file in sound_files: class OutboundSoundEffectWrapper(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, LLMFullResponseEndFrame): await self.push_frame(sounds["ding1.wav"]) # In case anything else downstream needs it @@ -72,8 +70,6 @@ class OutboundSoundEffectWrapper(FrameProcessor): class InboundSoundEffectWrapper(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, OpenAILLMContextFrame): await self.push_frame(sounds["ding2.wav"]) # In case anything else downstream needs it diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index b5bb577aa..3f00bafc9 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -42,8 +42,6 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index bc76afc73..52ddc6e43 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -42,8 +42,6 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index d8474b568..1840d7117 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -42,8 +42,6 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index bc6f5a4ea..f3690b277 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -42,8 +42,6 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/13-whisper-transcription.py b/examples/foundational/13-whisper-transcription.py index c895cb944..7a1657df7 100644 --- a/examples/foundational/13-whisper-transcription.py +++ b/examples/foundational/13-whisper-transcription.py @@ -30,8 +30,6 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13a-whisper-local.py b/examples/foundational/13a-whisper-local.py index c1ba37ca9..2d0b0f9d7 100644 --- a/examples/foundational/13a-whisper-local.py +++ b/examples/foundational/13a-whisper-local.py @@ -28,8 +28,6 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13b-deepgram-transcription.py b/examples/foundational/13b-deepgram-transcription.py index 7b3a25316..c915f9b42 100644 --- a/examples/foundational/13b-deepgram-transcription.py +++ b/examples/foundational/13b-deepgram-transcription.py @@ -31,8 +31,6 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13c-gladia-transcription.py b/examples/foundational/13c-gladia-transcription.py index acc21b6c2..13ef5556d 100644 --- a/examples/foundational/13c-gladia-transcription.py +++ b/examples/foundational/13c-gladia-transcription.py @@ -29,8 +29,6 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13d-assemblyai-transcription.py b/examples/foundational/13d-assemblyai-transcription.py index d10a80274..ea112b184 100644 --- a/examples/foundational/13d-assemblyai-transcription.py +++ b/examples/foundational/13d-assemblyai-transcription.py @@ -29,8 +29,6 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 2deeb3da4..e00714a75 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -64,7 +64,6 @@ class StatementJudgeContextFilter(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) # We must not block system frames. if isinstance(frame, SystemFrame): await self.push_frame(frame, direction) @@ -118,7 +117,6 @@ class CompletenessCheck(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) if isinstance(frame, TextFrame) and frame.text == "YES": logger.debug("Completeness check YES") await self.push_frame(UserStoppedSpeakingFrame()) @@ -141,8 +139,6 @@ class OutputGate(FrameProcessor): self._gate_open = True async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # We must not block system frames. if isinstance(frame, SystemFrame): if isinstance(frame, StartFrame): diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 97bc57ec1..3f12e5f34 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -101,12 +101,12 @@ HIGH PRIORITY SIGNALS: Examples: # Complete Wh-question -[{"role": "assistant", "content": "I can help you learn."}, +[{"role": "assistant", "content": "I can help you learn."}, {"role": "user", "content": "What's the fastest way to learn Spanish"}] Output: YES # Complete Yes/No question despite STT error -[{"role": "assistant", "content": "I know about planets."}, +[{"role": "assistant", "content": "I know about planets."}, {"role": "user", "content": "Is is Jupiter the biggest planet"}] Output: YES @@ -118,12 +118,12 @@ Output: YES Examples: # Direct instruction -[{"role": "assistant", "content": "I can explain many topics."}, +[{"role": "assistant", "content": "I can explain many topics."}, {"role": "user", "content": "Tell me about black holes"}] Output: YES # Action demand -[{"role": "assistant", "content": "I can help with math."}, +[{"role": "assistant", "content": "I can help with math."}, {"role": "user", "content": "Solve this equation x plus 5 equals 12"}] Output: YES @@ -134,12 +134,12 @@ Output: YES Examples: # Specific answer -[{"role": "assistant", "content": "What's your favorite color?"}, +[{"role": "assistant", "content": "What's your favorite color?"}, {"role": "user", "content": "I really like blue"}] Output: YES # Option selection -[{"role": "assistant", "content": "Would you prefer morning or evening?"}, +[{"role": "assistant", "content": "Would you prefer morning or evening?"}, {"role": "user", "content": "Morning"}] Output: YES @@ -153,17 +153,17 @@ MEDIUM PRIORITY SIGNALS: Examples: # Self-correction reaching completion -[{"role": "assistant", "content": "What would you like to know?"}, +[{"role": "assistant", "content": "What would you like to know?"}, {"role": "user", "content": "Tell me about... no wait, explain how rainbows form"}] Output: YES # Topic change with complete thought -[{"role": "assistant", "content": "The weather is nice today."}, +[{"role": "assistant", "content": "The weather is nice today."}, {"role": "user", "content": "Actually can you tell me who invented the telephone"}] Output: YES # Mid-sentence completion -[{"role": "assistant", "content": "Hello I'm ready."}, +[{"role": "assistant", "content": "Hello I'm ready."}, {"role": "user", "content": "What's the capital of? France"}] Output: YES @@ -175,12 +175,12 @@ Output: YES Examples: # Acknowledgment -[{"role": "assistant", "content": "Should we talk about history?"}, +[{"role": "assistant", "content": "Should we talk about history?"}, {"role": "user", "content": "Sure"}] Output: YES # Disagreement with completion -[{"role": "assistant", "content": "Is that what you meant?"}, +[{"role": "assistant", "content": "Is that what you meant?"}, {"role": "user", "content": "No not really"}] Output: YES @@ -194,12 +194,12 @@ LOW PRIORITY SIGNALS: Examples: # Word repetition but complete -[{"role": "assistant", "content": "I can help with that."}, +[{"role": "assistant", "content": "I can help with that."}, {"role": "user", "content": "What what is the time right now"}] Output: YES # Missing punctuation but complete -[{"role": "assistant", "content": "I can explain that."}, +[{"role": "assistant", "content": "I can explain that."}, {"role": "user", "content": "Please tell me how computers work"}] Output: YES @@ -211,12 +211,12 @@ Output: YES Examples: # Filler words but complete -[{"role": "assistant", "content": "What would you like to know?"}, +[{"role": "assistant", "content": "What would you like to know?"}, {"role": "user", "content": "Um uh how do airplanes fly"}] Output: YES # Thinking pause but incomplete -[{"role": "assistant", "content": "I can explain anything."}, +[{"role": "assistant", "content": "I can explain anything."}, {"role": "user", "content": "Well um I want to know about the"}] Output: NO @@ -241,17 +241,17 @@ DECISION RULES: Examples: # Incomplete despite corrections -[{"role": "assistant", "content": "What would you like to know about?"}, +[{"role": "assistant", "content": "What would you like to know about?"}, {"role": "user", "content": "Can you tell me about"}] Output: NO # Complete despite multiple artifacts -[{"role": "assistant", "content": "I can help you learn."}, +[{"role": "assistant", "content": "I can help you learn."}, {"role": "user", "content": "How do you I mean what's the best way to learn programming"}] Output: YES # Trailing off incomplete -[{"role": "assistant", "content": "I can explain anything."}, +[{"role": "assistant", "content": "I can explain anything."}, {"role": "user", "content": "I was wondering if you could tell me why"}] Output: NO """ @@ -268,7 +268,6 @@ class StatementJudgeContextFilter(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) # We must not block system frames. if isinstance(frame, SystemFrame): await self.push_frame(frame, direction) @@ -320,8 +319,6 @@ class CompletenessCheck(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame) and frame.text == "YES": logger.debug("!!! Completeness check YES") await self.push_frame(UserStoppedSpeakingFrame()) @@ -344,8 +341,6 @@ class OutputGate(FrameProcessor): self._gate_open = True async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # We must not block system frames. if isinstance(frame, SystemFrame): if isinstance(frame, StartFrame): diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 1ff8aa23e..e506372a5 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -90,8 +90,6 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): self._user_speaking = False async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # ignore context frame if isinstance(frame, OpenAILLMContextFrame): return @@ -133,8 +131,6 @@ class CompletenessCheck(FrameProcessor): self._audio_accumulator = audio_accumulator async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame) and frame.text.startswith("YES"): logger.debug("Completeness check YES") await self.push_frame(UserStoppedSpeakingFrame()) @@ -159,8 +155,6 @@ class OutputGate(FrameProcessor): self._gate_open = True async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # We must not block system frames. if isinstance(frame, SystemFrame): if isinstance(frame, StartFrame): diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index 843d24e1f..abeb62043 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -95,8 +95,6 @@ class UserAudioCollector(FrameProcessor): self._user_speaking = False async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if isinstance(frame, TranscriptionFrame): # We could gracefully handle both audio input and text/transcription input ... # but let's leave that as an exercise to the reader. :-) @@ -135,8 +133,6 @@ class InputTranscriptionContextFilter(FrameProcessor): """ async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if isinstance(frame, SystemFrame): # We don't want to block system frames. await self.push_frame(frame, direction) @@ -210,8 +206,6 @@ class InputTranscriptionFrameEmitter(FrameProcessor): self._aggregation = "" async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): self._aggregation += frame.text elif isinstance(frame, LLMFullResponseEndFrame): @@ -262,8 +256,6 @@ class TranscriptionContextFixup(FrameProcessor): audio_part.text = self._transcript async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if isinstance(frame, LLMDemoTranscriptionFrame): logger.info(f"Transcription from Gemini: {frame.text}") self._transcript = frame.text diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 54c2013b4..1c412e88a 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -81,8 +81,6 @@ class TalkingAnimation(FrameProcessor): self._is_talking = False async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, BotStartedSpeakingFrame): if not self._is_talking: await self.push_frame(talking_frame) @@ -103,8 +101,6 @@ class UserImageRequester(FrameProcessor): self.participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if self.participant_id and isinstance(frame, TextFrame): if frame.text == user_request_answer: await self.push_frame( @@ -121,8 +117,6 @@ class TextFilterProcessor(FrameProcessor): self.text = text async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): if frame.text != self.text: await self.push_frame(frame) @@ -132,8 +126,6 @@ class TextFilterProcessor(FrameProcessor): class ImageFilterProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if not isinstance(frame, ImageRawFrame): await self.push_frame(frame, direction) diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index 991df1cd1..0ce46e50e 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -95,8 +95,6 @@ class TalkingAnimation(FrameProcessor): frame: The incoming frame to process direction: The direction of frame flow in the pipeline """ - await super().process_frame(frame, direction) - # Switch to talking animation when bot starts speaking if isinstance(frame, BotStartedSpeakingFrame): if not self._is_talking: diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index a3a68c839..02685a99b 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -95,8 +95,6 @@ class TalkingAnimation(FrameProcessor): frame: The incoming frame to process direction: The direction of frame flow in the pipeline """ - await super().process_frame(frame, direction) - # Switch to talking animation when bot starts speaking if isinstance(frame, BotStartedSpeakingFrame): if not self._is_talking: diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/src/processors.py index 6aa9ad7ab..dd46f9c82 100644 --- a/examples/storytelling-chatbot/src/processors.py +++ b/examples/storytelling-chatbot/src/processors.py @@ -54,8 +54,6 @@ class StoryImageProcessor(FrameProcessor): self._fal_service = fal_service async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, StoryImageFrame): try: async with timeout(7): @@ -90,8 +88,6 @@ class StoryProcessor(FrameProcessor): self._story = story async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, UserStoppedSpeakingFrame): # Send an app message to the UI await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN)) diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index e654c0159..59f495360 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -51,8 +51,6 @@ class TranslationProcessor(FrameProcessor): self._language = language async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): context = [ { @@ -78,8 +76,6 @@ class TranslationSubtitles(FrameProcessor): # subtitles. # async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): message = {"language": self._language, "text": frame.text} await self.push_frame(DailyTransportMessageFrame(message)) diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 40bfea90d..7499192fb 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -28,8 +28,6 @@ class Source(FrameProcessor): self._push_frame_func = push_frame_func async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - match direction: case FrameDirection.UPSTREAM: if isinstance(frame, SystemFrame): @@ -51,8 +49,6 @@ class Sink(FrameProcessor): self._push_frame_func = push_frame_func async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - match direction: case FrameDirection.UPSTREAM: await self.push_frame(frame, direction) @@ -120,8 +116,6 @@ class ParallelPipeline(BasePipeline): self._down_task = loop.create_task(self._process_down_queue()) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, StartFrame): await self._start_tasks() diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py index 703f911fe..457b70cab 100644 --- a/src/pipecat/pipeline/pipeline.py +++ b/src/pipecat/pipeline/pipeline.py @@ -17,8 +17,6 @@ class PipelineSource(FrameProcessor): self._upstream_push_frame = upstream_push_frame async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - match direction: case FrameDirection.UPSTREAM: await self._upstream_push_frame(frame, direction) @@ -32,8 +30,6 @@ class PipelineSink(FrameProcessor): self._downstream_push_frame = downstream_push_frame async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - match direction: case FrameDirection.UPSTREAM: await self.push_frame(frame, direction) @@ -74,8 +70,6 @@ class Pipeline(BasePipeline): await self._cleanup_processors() async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if direction == FrameDirection.DOWNSTREAM: await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM) elif direction == FrameDirection.UPSTREAM: diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py index 20f4275e4..4dcf190de 100644 --- a/src/pipecat/pipeline/sync_parallel_pipeline.py +++ b/src/pipecat/pipeline/sync_parallel_pipeline.py @@ -31,8 +31,6 @@ class Source(FrameProcessor): self._up_queue = upstream_queue async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - match direction: case FrameDirection.UPSTREAM: await self._up_queue.put(frame) @@ -46,8 +44,6 @@ class Sink(FrameProcessor): self._down_queue = downstream_queue async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - match direction: case FrameDirection.UPSTREAM: await self.push_frame(frame, direction) @@ -103,8 +99,6 @@ class SyncParallelPipeline(BasePipeline): # async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # The last processor of each pipeline needs to be synchronous otherwise # this element won't work. Since, we know it should be synchronous we # push a SyncFrame. Since frames are ordered we know this frame will be diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index f09013a58..d8bada663 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -45,8 +45,6 @@ class Source(FrameProcessor): self._up_queue = up_queue async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - match direction: case FrameDirection.UPSTREAM: await self._handle_upstream_frame(frame) @@ -75,8 +73,6 @@ class Sink(FrameProcessor): self._down_queue = down_queue async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # We really just want to know when the EndFrame reached the sink. if isinstance(frame, EndFrame): await self._down_queue.put(frame) diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index c39a35c82..763dc456c 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -56,8 +56,6 @@ class GatedAggregator(FrameProcessor): self._accumulator: List[Tuple[Frame, FrameDirection]] = [] async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # We must not block system frames. if isinstance(frame, SystemFrame): await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index 71a540dd4..9b0d77d32 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -24,8 +24,6 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): self._last_context_frame = None async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, StartFrame): await self.push_frame(frame) await self._start() diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 479746471..544b49dda 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -86,8 +86,6 @@ class LLMResponseAggregator(FrameProcessor): # and T2 would be dropped. async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - send_aggregation = False if isinstance(frame, self._start_frame): @@ -240,8 +238,6 @@ class LLMFullResponseAggregator(FrameProcessor): self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): self._aggregation += frame.text elif isinstance(frame, LLMFullResponseEndFrame): diff --git a/src/pipecat/processors/aggregators/sentence.py b/src/pipecat/processors/aggregators/sentence.py index d0c593a83..ab400b2a0 100644 --- a/src/pipecat/processors/aggregators/sentence.py +++ b/src/pipecat/processors/aggregators/sentence.py @@ -33,8 +33,6 @@ class SentenceAggregator(FrameProcessor): self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # We ignore interim description at this point. if isinstance(frame, InterimTranscriptionFrame): return diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index 903019059..78287127f 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -85,8 +85,6 @@ class ResponseAggregator(FrameProcessor): # and T2 would be dropped. async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - send_aggregation = False if isinstance(frame, self._start_frame): diff --git a/src/pipecat/processors/aggregators/vision_image_frame.py b/src/pipecat/processors/aggregators/vision_image_frame.py index d07337f06..3a4eda330 100644 --- a/src/pipecat/processors/aggregators/vision_image_frame.py +++ b/src/pipecat/processors/aggregators/vision_image_frame.py @@ -31,8 +31,6 @@ class VisionImageFrameAggregator(FrameProcessor): self._describe_text = None async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): self._describe_text = frame.text elif isinstance(frame, InputImageRawFrame): diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py index 4f9bc85d0..356ef4388 100644 --- a/src/pipecat/processors/async_generator.py +++ b/src/pipecat/processors/async_generator.py @@ -24,8 +24,6 @@ class AsyncGeneratorProcessor(FrameProcessor): self._data_queue = asyncio.Queue() async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, (CancelFrame, EndFrame)): diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 488a251f0..c7bb36736 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -68,8 +68,6 @@ class AudioBufferProcessor(FrameProcessor): self._bot_audio_buffer = bytearray() async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # Include all audio from the user. if isinstance(frame, InputAudioRawFrame): resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate) diff --git a/src/pipecat/processors/audio/vad/silero.py b/src/pipecat/processors/audio/vad/silero.py index 4aa32a163..1db510f24 100644 --- a/src/pipecat/processors/audio/vad/silero.py +++ b/src/pipecat/processors/audio/vad/silero.py @@ -39,8 +39,6 @@ class SileroVAD(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, AudioRawFrame): await self._analyze_audio(frame) if self._audio_passthrough: diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py index 11f2e601a..e87034a1a 100644 --- a/src/pipecat/processors/filters/frame_filter.py +++ b/src/pipecat/processors/filters/frame_filter.py @@ -26,7 +26,5 @@ class FrameFilter(FrameProcessor): return isinstance(frame, ControlFrame) or isinstance(frame, SystemFrame) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if self._should_passthrough_frame(frame): await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index e38cea3e0..522a89324 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -29,8 +29,6 @@ class FunctionFilter(FrameProcessor): return isinstance(frame, SystemFrame) or direction != self._direction async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - passthrough = self._should_passthrough_frame(frame, direction) allowed = await self._filter(frame) if passthrough or allowed: diff --git a/src/pipecat/processors/filters/identity_filter.py b/src/pipecat/processors/filters/identity_filter.py index d6f896b73..c837e02a7 100644 --- a/src/pipecat/processors/filters/identity_filter.py +++ b/src/pipecat/processors/filters/identity_filter.py @@ -26,5 +26,4 @@ class IdentityFilter(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): """Process an incoming frame by passing it through unchanged.""" - await super().process_frame(frame, direction) await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index f1a7afbef..441f32fb9 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -45,8 +45,6 @@ class WakeCheckFilter(FrameProcessor): self._wake_patterns.append(pattern) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - try: if isinstance(frame, TranscriptionFrame): p = self._participant_states.get(frame.user_id) diff --git a/src/pipecat/processors/filters/wake_notifier_filter.py b/src/pipecat/processors/filters/wake_notifier_filter.py index a7f074ccb..7623b6da8 100644 --- a/src/pipecat/processors/filters/wake_notifier_filter.py +++ b/src/pipecat/processors/filters/wake_notifier_filter.py @@ -32,8 +32,6 @@ class WakeNotifierFilter(FrameProcessor): self._filter = filter async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, self._types) and await self._filter(frame): await self._notifier.notify() diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 52066b4f4..e0806a0bf 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -161,6 +161,13 @@ class FrameProcessor: def get_clock(self) -> BaseClock: return self._clock + async def pause_processing_frames(self): + self.__should_block_frames = True + + async def resume_processing_frames(self): + self.__input_event.set() + self.__should_block_frames = False + async def queue_frame( self, frame: Frame, @@ -175,32 +182,13 @@ class FrameProcessor: if isinstance(frame, SystemFrame): # We don't want to queue system frames. - await self.process_frame(frame, direction) + await self._process_frame(frame, direction) else: # We queue everything else. await self.__input_queue.put((frame, direction, callback)) - async def pause_processing_frames(self): - self.__should_block_frames = True - - async def resume_processing_frames(self): - self.__input_event.set() - self.__should_block_frames = False - async def process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, StartFrame): - self._clock = frame.clock - self._allow_interruptions = frame.allow_interruptions - self._enable_metrics = frame.enable_metrics - self._enable_usage_metrics = frame.enable_usage_metrics - self._report_only_initial_ttfb = frame.report_only_initial_ttfb - elif isinstance(frame, StartInterruptionFrame): - await self._start_interruption() - await self.stop_all_metrics() - elif isinstance(frame, StopInterruptionFrame): - self._should_report_ttfb = True - elif isinstance(frame, CancelFrame): - self._cancelling = True + pass async def push_error(self, error: ErrorFrame): await self.push_frame(error, FrameDirection.UPSTREAM) @@ -228,6 +216,28 @@ class FrameProcessor: raise Exception(f"Event handler {event_name} already registered") self._event_handlers[event_name] = [] + # + # Frame processing + # + + async def _process_frame(self, frame: Frame, direction: FrameDirection): + if isinstance(frame, StartFrame): + self._clock = frame.clock + self._allow_interruptions = frame.allow_interruptions + self._enable_metrics = frame.enable_metrics + self._enable_usage_metrics = frame.enable_usage_metrics + self._report_only_initial_ttfb = frame.report_only_initial_ttfb + elif isinstance(frame, StartInterruptionFrame): + await self._start_interruption() + await self.stop_all_metrics() + elif isinstance(frame, StopInterruptionFrame): + self._should_report_ttfb = True + elif isinstance(frame, CancelFrame): + self._cancelling = True + + # Call subclass. + await self.process_frame(frame, direction) + # # Handle interruptions # @@ -289,7 +299,7 @@ class FrameProcessor: (frame, direction, callback) = await self.__input_queue.get() # Process the frame. - await self.process_frame(frame, direction) + await self._process_frame(frame, direction) # If this frame has an associated callback, call it now. if callback: diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index c0b657244..25de11070 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -36,8 +36,6 @@ class LangchainProcessor(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, LLMMessagesFrame): # Messages are accumulated on the context as a list of messages. # The last one by the human is the one we want to send to the LLM. diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 471bdbb88..b91abb181 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -380,8 +380,6 @@ class RTVISpeakingProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)): @@ -415,8 +413,6 @@ class RTVIUserTranscriptionProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)): @@ -446,8 +442,6 @@ class RTVIUserLLMTextProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, OpenAILLMContextFrame): @@ -473,8 +467,6 @@ class RTVIBotTranscriptionProcessor(RTVIFrameProcessor): self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): @@ -496,8 +488,6 @@ class RTVIBotLLMProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, LLMFullResponseStartFrame): @@ -514,8 +504,6 @@ class RTVIBotTTSProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, TTSStartedFrame): @@ -532,8 +520,6 @@ class RTVIMetricsProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) if isinstance(frame, MetricsFrame): @@ -642,8 +628,6 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message, exclude_none=False) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # Specific system frames if isinstance(frame, StartFrame): # Push StartFrame before start(), because we want StartFrame to be diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 649a2c529..09456f12e 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -66,8 +66,6 @@ class GStreamerPipelineSource(FrameProcessor): bus.connect("message", self._on_gstreamer_message) async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # Specific system frames if isinstance(frame, StartFrame): # Push StartFrame before start(), because we want StartFrame to be diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index e674b6b84..80902ee59 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -35,8 +35,6 @@ class IdleFrameProcessor(FrameProcessor): self._create_idle_task() async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - await self.push_frame(frame, direction) # If we are not waiting for any specific frame set the event, otherwise diff --git a/src/pipecat/processors/text_transformer.py b/src/pipecat/processors/text_transformer.py index 79e9b885e..90ef6b8bc 100644 --- a/src/pipecat/processors/text_transformer.py +++ b/src/pipecat/processors/text_transformer.py @@ -27,8 +27,6 @@ class StatelessTextTransformer(FrameProcessor): self._transform_fn = transform_fn async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame): result = self._transform_fn(frame.text) if isinstance(result, Coroutine): diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 160c49908..91cbd2334 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -43,8 +43,6 @@ class UserIdleProcessor(FrameProcessor): await self._idle_task async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # Check for end frames before processing if isinstance(frame, (EndFrame, CancelFrame)): await self._stop() diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index e0f16e220..e324d413c 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -110,8 +110,6 @@ class AIService(FrameProcessor): logger.warning(f"Unknown setting for {self.name} service: {key}") async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, StartFrame): await self.start(frame) elif isinstance(frame, CancelFrame): diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index bfae861dc..e61fb394c 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -92,7 +92,6 @@ class SimliVideoService(FrameProcessor): pass async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) if isinstance(frame, StartFrame): await self.push_frame(frame, direction) await self._start_connection() diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 025a5bed2..64583c758 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -79,8 +79,6 @@ class BaseInputTransport(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # Specific system frames if isinstance(frame, StartFrame): # Push StartFrame before start(), because we want StartFrame to be diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index b25b4c78c..a9154f4dc 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -108,8 +108,6 @@ class BaseOutputTransport(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - # # System frames (like StartInterruptionFrame) are pushed # immediately. Other frames require order so they are put in the sink diff --git a/src/pipecat/utils/test_frame_processor.py b/src/pipecat/utils/test_frame_processor.py index e46bae7ad..ec37efd80 100644 --- a/src/pipecat/utils/test_frame_processor.py +++ b/src/pipecat/utils/test_frame_processor.py @@ -13,8 +13,6 @@ class TestFrameProcessor(FrameProcessor): super().__init__() async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if not self.test_frames[ 0 ]: # then we've run out of required frames but the generator is still going? diff --git a/tests/test_langchain.py b/tests/test_langchain.py index d30d213bd..b1f8f618d 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -42,8 +42,6 @@ class TestLangchain(unittest.IsolatedAsyncioTestCase): return self.name async def process_frame(self, frame, direction): - await super().process_frame(frame, direction) - if isinstance(frame, LLMFullResponseStartFrame): self.start_collecting = True elif isinstance(frame, TextFrame) and self.start_collecting: From 19c178ebc740d70676a154fb99bb2b60b7124c77 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 16:11:48 -0500 Subject: [PATCH 12/90] Fix docs generation build issues --- .readthedocs.yaml | 44 +++---------------- docs/api/README.md | 2 +- docs/api/conf.py | 12 +++-- docs/api/requirements-playht.txt | 3 -- docs/api/requirements-riva.txt | 3 -- ...requirements-base.txt => requirements.txt} | 6 ++- docs/api/rtd-test.sh | 24 +++++----- src/pipecat/services/riva.py | 2 +- 8 files changed, 34 insertions(+), 62 deletions(-) delete mode 100644 docs/api/requirements-playht.txt delete mode 100644 docs/api/requirements-riva.txt rename docs/api/{requirements-base.txt => requirements.txt} (83%) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 379e31465..c9176cdda 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -4,58 +4,26 @@ build: os: ubuntu-22.04 tools: python: '3.12' + apt_packages: + - portaudio19-dev + - python3-dev + - libasound2-dev jobs: pre_build: - # Commands to run before the build - python -m pip install --upgrade pip - pip install wheel setuptools post_build: - # Commands to run after the build - echo "Build completed" sphinx: configuration: docs/api/conf.py - fail_on_warning: false # Set to true if you want builds to fail on warnings + fail_on_warning: false python: install: - - requirements: docs/api/requirements-base.txt - # Try to install Riva first, fall back to PlayHT if it fails - - requirements: docs/api/requirements-riva.txt || true - - requirements: docs/api/requirements-playht.txt || true + - requirements: docs/api/requirements.txt - method: pip path: . - extra_requirements: - - anthropic - - assemblyai - - aws - - azure - - canonical - - cartesia - - daily - - deepgram - - elevenlabs - - fal - - fireworks - - gladia - - google - - grok - - groq - - krisp - - langchain - - livekit - - lmnt - - local - - moondream - - nim - - noisereduce - - openai - - openpipe - - silero - - simli - - soundfile - - websocket - - whisper search: ranking: diff --git a/docs/api/README.md b/docs/api/README.md index 392430071..22b62d45e 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -7,7 +7,7 @@ This directory contains the source files for auto-generating Pipecat's server AP 1. Install documentation dependencies: ```bash -pip install -r requirements-base.txt requirements-playht.txt requirements-riva.txt +pip install -r requirements.txt ``` 2. Make the build scripts executable: diff --git a/docs/api/conf.py b/docs/api/conf.py index 51e000cbf..36b4f4df0 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -44,7 +44,12 @@ autodoc_default_options = { autodoc_mock_imports = [ "riva", "livekit", - "pyht", + "pyht", # Base PlayHT package + "pyht.async_client", # PlayHT specific imports + "pyht.client", + "pyht.protos", + "pyht.protos.api_pb2", + "pipecat_ai_playht", # PlayHT wrapper "anthropic", "assemblyai", "boto3", @@ -63,13 +68,14 @@ autodoc_mock_imports = [ "openpipe", "simli", "soundfile", - # Add these new mocks + # Existing mocks + "pipecat_ai_krisp", "pyaudio", "_tkinter", "tkinter", "daily", "daily_python", - "pydantic.BaseModel", # Mock base pydantic to avoid model conflicts + "pydantic.BaseModel", "pydantic.Field", "pydantic._internal._model_construction", "pydantic._internal._fields", diff --git a/docs/api/requirements-playht.txt b/docs/api/requirements-playht.txt deleted file mode 100644 index 1f0bc24ea..000000000 --- a/docs/api/requirements-playht.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Force specific grpcio version for PlayHT -grpcio>=1.68.0 -pipecat-ai[playht] \ No newline at end of file diff --git a/docs/api/requirements-riva.txt b/docs/api/requirements-riva.txt deleted file mode 100644 index 6bd4c69f9..000000000 --- a/docs/api/requirements-riva.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Force specific grpcio version for Riva -grpcio==1.65.4 -pipecat-ai[riva] \ No newline at end of file diff --git a/docs/api/requirements-base.txt b/docs/api/requirements.txt similarity index 83% rename from docs/api/requirements-base.txt rename to docs/api/requirements.txt index ff397d0ef..afb4c8dcf 100644 --- a/docs/api/requirements-base.txt +++ b/docs/api/requirements.txt @@ -21,7 +21,7 @@ pipecat-ai[gladia] pipecat-ai[google] pipecat-ai[grok] pipecat-ai[groq] -pipecat-ai[krisp] +# pipecat-ai[krisp] # Mocked instead pipecat-ai[langchain] pipecat-ai[livekit] pipecat-ai[lmnt] @@ -30,7 +30,9 @@ pipecat-ai[moondream] pipecat-ai[nim] pipecat-ai[noisereduce] pipecat-ai[openai] -pipecat-ai[openpipe] +# pipecat-ai[openpipe] +# pipecat-ai[playht] # Mocked due to grpcio conflict with riva +pipecat-ai[riva] pipecat-ai[silero] pipecat-ai[simli] pipecat-ai[soundfile] diff --git a/docs/api/rtd-test.sh b/docs/api/rtd-test.sh index 87d8880b4..2b2c30d5d 100755 --- a/docs/api/rtd-test.sh +++ b/docs/api/rtd-test.sh @@ -10,20 +10,15 @@ echo "Creating test directory: $TEST_DIR" mkdir -p "$TEST_DIR" cd "$TEST_DIR" -# Create single virtual environment +# Create virtual environment python -m venv venv source venv/bin/activate -echo "Installing base dependencies..." +echo "Installing build dependencies..." pip install --upgrade pip wheel setuptools -pip install -r "$DOCS_DIR/requirements-base.txt" -# Try to install optional dependencies, but don't fail if they don't work -echo "Installing Riva dependencies..." -pip install -r "$DOCS_DIR/requirements-riva.txt" || echo "Failed to install Riva dependencies" - -echo "Installing PlayHT dependencies..." -pip install -r "$DOCS_DIR/requirements-playht.txt" || echo "Failed to install PlayHT dependencies" +echo "Installing documentation dependencies..." +pip install -r "$DOCS_DIR/requirements.txt" echo "Building documentation..." cd "$DOCS_DIR" @@ -31,6 +26,13 @@ sphinx-build -b html . "_build/html" echo "Build complete. Check _build/html directory for output." +# Print summary +echo -e "\n=== Build Summary ===" +echo "Documentation: $DOCS_DIR/_build/html" +echo "Test environment: $TEST_DIR" +echo -e "\nTo view the documentation:" +echo "open $DOCS_DIR/_build/html/index.html" + # Print installed packages for verification -echo "Installed packages:" -pip freeze \ No newline at end of file +echo -e "\n=== Installed Packages ===" +pip freeze | grep -E "sphinx|pipecat" \ No newline at end of file diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index f57372775..6be722d49 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -8,7 +8,7 @@ import asyncio from typing import AsyncGenerator, Optional from loguru import logger -from pydantic.main import BaseModel +from pydantic import BaseModel from pipecat.frames.frames import ( CancelFrame, From ec6e71c8eae6866e6cface3a21ad07f6f2fd5a1e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 18:08:24 -0500 Subject: [PATCH 13/90] Add docs badge to README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b68d62778..86e3ddd3e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@  pipecat -[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) +[![PyPI](https://img.shields.io/pypi/v/pipecat-ai)](https://pypi.org/project/pipecat-ai) [![Docs](https://img.shields.io/badge/Documentation-blue)](https://docs.pipecat.ai) [![Discord](https://img.shields.io/discord/1239284677165056021)](https://discord.gg/pipecat) Pipecat is an open source Python framework for building voice and multimodal conversational agents. It handles the complex orchestration of AI services, network transport, audio processing, and multimodal interactions, letting you focus on creating engaging experiences. From 6d11911d835be7fe2c144d9fa38f845f644660f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 12 Dec 2024 17:03:40 -0800 Subject: [PATCH 14/90] Revert "no longer necessary to call super().process_frame(frame, direction)" --- CHANGELOG.md | 6 --- .../foundational/05-sync-speech-and-image.py | 2 + .../05a-local-sync-speech-and-image.py | 6 +++ examples/foundational/06a-image-sync.py | 2 + .../07s-interruptible-google-audio-in.py | 5 ++ examples/foundational/09-mirror.py | 2 + examples/foundational/09a-local-mirror.py | 2 + examples/foundational/11-sound-effects.py | 4 ++ examples/foundational/12-describe-video.py | 2 + .../12a-describe-video-gemini-flash.py | 2 + .../foundational/12b-describe-video-gpt-4o.py | 2 + .../12c-describe-video-anthropic.py | 2 + .../foundational/13-whisper-transcription.py | 2 + examples/foundational/13a-whisper-local.py | 2 + .../13b-deepgram-transcription.py | 2 + .../foundational/13c-gladia-transcription.py | 2 + .../13d-assemblyai-transcription.py | 2 + .../22b-natural-conversation-proposal.py | 4 ++ .../22c-natural-conversation-mixed-llms.py | 41 +++++++------- .../22d-natural-conversation-gemini-audio.py | 6 +++ examples/foundational/25-google-audio-in.py | 8 +++ examples/moondream-chatbot/bot.py | 8 +++ examples/simple-chatbot/server/bot-gemini.py | 2 + examples/simple-chatbot/server/bot-openai.py | 2 + .../storytelling-chatbot/src/processors.py | 4 ++ examples/translation-chatbot/bot.py | 4 ++ src/pipecat/pipeline/parallel_pipeline.py | 6 +++ src/pipecat/pipeline/pipeline.py | 6 +++ .../pipeline/sync_parallel_pipeline.py | 6 +++ src/pipecat/pipeline/task.py | 4 ++ src/pipecat/processors/aggregators/gated.py | 2 + .../aggregators/gated_openai_llm_context.py | 2 + .../processors/aggregators/llm_response.py | 4 ++ .../processors/aggregators/sentence.py | 2 + .../processors/aggregators/user_response.py | 2 + .../aggregators/vision_image_frame.py | 2 + src/pipecat/processors/async_generator.py | 2 + .../audio/audio_buffer_processor.py | 2 + src/pipecat/processors/audio/vad/silero.py | 2 + .../processors/filters/frame_filter.py | 2 + .../processors/filters/function_filter.py | 2 + .../processors/filters/identity_filter.py | 1 + .../processors/filters/wake_check_filter.py | 2 + .../filters/wake_notifier_filter.py | 2 + src/pipecat/processors/frame_processor.py | 54 ++++++++----------- .../processors/frameworks/langchain.py | 2 + src/pipecat/processors/frameworks/rtvi.py | 16 ++++++ .../processors/gstreamer/pipeline_source.py | 2 + .../processors/idle_frame_processor.py | 2 + src/pipecat/processors/text_transformer.py | 2 + src/pipecat/processors/user_idle_processor.py | 2 + src/pipecat/services/ai_services.py | 2 + src/pipecat/services/simli.py | 1 + src/pipecat/transports/base_input.py | 2 + src/pipecat/transports/base_output.py | 2 + src/pipecat/utils/test_frame_processor.py | 2 + tests/test_langchain.py | 2 + 57 files changed, 212 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4a005280..5ca41b8a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,12 +13,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa). -### Changed - -- It's no longer necessary to call `super().process_frame(frame, direction)` if - you subclass and implement `FrameProcessor.process_frame()`. This is all now - done internally and will avoid possible issues if you forget to add it. - ### Deprecated - `AWSTTSService` is now deprecated, use `PollyTTSService` instead. diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 8d5790ac7..64f85930b 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -56,6 +56,8 @@ class MonthPrepender(FrameProcessor): self.prepend_to_next_text_frame = False async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, MonthFrame): self.most_recent_month = frame.month elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame): diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index f6e5f0ce6..4a561c073 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -62,6 +62,8 @@ async def main(): self.text = "" async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): self.text = frame.text await self.push_frame(frame, direction) @@ -73,6 +75,8 @@ async def main(): self.frame = None async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TTSAudioRawFrame): self.audio.extend(frame.audio) self.frame = OutputAudioRawFrame( @@ -86,6 +90,8 @@ async def main(): self.frame = None async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, URLImageRawFrame): self.frame = frame await self.push_frame(frame, direction) diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 11c894478..eda3c61df 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -47,6 +47,8 @@ class ImageSyncAggregator(FrameProcessor): self._waiting_image_bytes = self._waiting_image.tobytes() async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if not isinstance(frame, SystemFrame) and direction == FrameDirection.DOWNSTREAM: await self.push_frame( OutputImageRawFrame( diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 1db8b0d36..1778e0c62 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -82,6 +82,8 @@ class UserAudioCollector(FrameProcessor): self._user_speaking = False async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if isinstance(frame, TranscriptionFrame): # We could gracefully handle both audio input and text/transcription input ... # but let's leave that as an exercise to the reader. :-) @@ -124,6 +126,7 @@ class TranscriptExtractor(FrameProcessor): self._accumulating_transcript = False async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) if isinstance(frame, LLMFullResponseStartFrame): self._processing_llm_response = True self._accumulating_transcript = True @@ -177,6 +180,8 @@ class TanscriptionContextFixup(FrameProcessor): self._context.messages[-1].parts[-1].text += f"\n\n{marker}\n{self._transcript}\n" async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if isinstance(frame, MagicDemoTranscriptionFrame): self._transcript = frame.text elif isinstance(frame, LLMFullResponseEndFrame) or isinstance( diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index 8eaee5750..a719d54f6 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -35,6 +35,8 @@ logger.add(sys.stderr, level="DEBUG") class MirrorProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, InputAudioRawFrame): await self.push_frame( OutputAudioRawFrame( diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index 4a4a1fee1..539cca600 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -39,6 +39,8 @@ logger.add(sys.stderr, level="DEBUG") class MirrorProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, InputAudioRawFrame): await self.push_frame( OutputAudioRawFrame( diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index 50d3f9e33..d8692a7f1 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -60,6 +60,8 @@ for file in sound_files: class OutboundSoundEffectWrapper(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, LLMFullResponseEndFrame): await self.push_frame(sounds["ding1.wav"]) # In case anything else downstream needs it @@ -70,6 +72,8 @@ class OutboundSoundEffectWrapper(FrameProcessor): class InboundSoundEffectWrapper(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, OpenAILLMContextFrame): await self.push_frame(sounds["ding2.wav"]) # In case anything else downstream needs it diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index 3f00bafc9..b5bb577aa 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index 52ddc6e43..bc76afc73 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index 1840d7117..d8474b568 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index f3690b277..bc6f5a4ea 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -42,6 +42,8 @@ class UserImageRequester(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM diff --git a/examples/foundational/13-whisper-transcription.py b/examples/foundational/13-whisper-transcription.py index 7a1657df7..c895cb944 100644 --- a/examples/foundational/13-whisper-transcription.py +++ b/examples/foundational/13-whisper-transcription.py @@ -30,6 +30,8 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13a-whisper-local.py b/examples/foundational/13a-whisper-local.py index 2d0b0f9d7..c1ba37ca9 100644 --- a/examples/foundational/13a-whisper-local.py +++ b/examples/foundational/13a-whisper-local.py @@ -28,6 +28,8 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13b-deepgram-transcription.py b/examples/foundational/13b-deepgram-transcription.py index c915f9b42..7b3a25316 100644 --- a/examples/foundational/13b-deepgram-transcription.py +++ b/examples/foundational/13b-deepgram-transcription.py @@ -31,6 +31,8 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13c-gladia-transcription.py b/examples/foundational/13c-gladia-transcription.py index 13ef5556d..acc21b6c2 100644 --- a/examples/foundational/13c-gladia-transcription.py +++ b/examples/foundational/13c-gladia-transcription.py @@ -29,6 +29,8 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/13d-assemblyai-transcription.py b/examples/foundational/13d-assemblyai-transcription.py index ea112b184..d10a80274 100644 --- a/examples/foundational/13d-assemblyai-transcription.py +++ b/examples/foundational/13d-assemblyai-transcription.py @@ -29,6 +29,8 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptionLogger(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TranscriptionFrame): print(f"Transcription: {frame.text}") diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index e00714a75..2deeb3da4 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -64,6 +64,7 @@ class StatementJudgeContextFilter(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) # We must not block system frames. if isinstance(frame, SystemFrame): await self.push_frame(frame, direction) @@ -117,6 +118,7 @@ class CompletenessCheck(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) if isinstance(frame, TextFrame) and frame.text == "YES": logger.debug("Completeness check YES") await self.push_frame(UserStoppedSpeakingFrame()) @@ -139,6 +141,8 @@ class OutputGate(FrameProcessor): self._gate_open = True async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # We must not block system frames. if isinstance(frame, SystemFrame): if isinstance(frame, StartFrame): diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 3f12e5f34..97bc57ec1 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -101,12 +101,12 @@ HIGH PRIORITY SIGNALS: Examples: # Complete Wh-question -[{"role": "assistant", "content": "I can help you learn."}, +[{"role": "assistant", "content": "I can help you learn."}, {"role": "user", "content": "What's the fastest way to learn Spanish"}] Output: YES # Complete Yes/No question despite STT error -[{"role": "assistant", "content": "I know about planets."}, +[{"role": "assistant", "content": "I know about planets."}, {"role": "user", "content": "Is is Jupiter the biggest planet"}] Output: YES @@ -118,12 +118,12 @@ Output: YES Examples: # Direct instruction -[{"role": "assistant", "content": "I can explain many topics."}, +[{"role": "assistant", "content": "I can explain many topics."}, {"role": "user", "content": "Tell me about black holes"}] Output: YES # Action demand -[{"role": "assistant", "content": "I can help with math."}, +[{"role": "assistant", "content": "I can help with math."}, {"role": "user", "content": "Solve this equation x plus 5 equals 12"}] Output: YES @@ -134,12 +134,12 @@ Output: YES Examples: # Specific answer -[{"role": "assistant", "content": "What's your favorite color?"}, +[{"role": "assistant", "content": "What's your favorite color?"}, {"role": "user", "content": "I really like blue"}] Output: YES # Option selection -[{"role": "assistant", "content": "Would you prefer morning or evening?"}, +[{"role": "assistant", "content": "Would you prefer morning or evening?"}, {"role": "user", "content": "Morning"}] Output: YES @@ -153,17 +153,17 @@ MEDIUM PRIORITY SIGNALS: Examples: # Self-correction reaching completion -[{"role": "assistant", "content": "What would you like to know?"}, +[{"role": "assistant", "content": "What would you like to know?"}, {"role": "user", "content": "Tell me about... no wait, explain how rainbows form"}] Output: YES # Topic change with complete thought -[{"role": "assistant", "content": "The weather is nice today."}, +[{"role": "assistant", "content": "The weather is nice today."}, {"role": "user", "content": "Actually can you tell me who invented the telephone"}] Output: YES # Mid-sentence completion -[{"role": "assistant", "content": "Hello I'm ready."}, +[{"role": "assistant", "content": "Hello I'm ready."}, {"role": "user", "content": "What's the capital of? France"}] Output: YES @@ -175,12 +175,12 @@ Output: YES Examples: # Acknowledgment -[{"role": "assistant", "content": "Should we talk about history?"}, +[{"role": "assistant", "content": "Should we talk about history?"}, {"role": "user", "content": "Sure"}] Output: YES # Disagreement with completion -[{"role": "assistant", "content": "Is that what you meant?"}, +[{"role": "assistant", "content": "Is that what you meant?"}, {"role": "user", "content": "No not really"}] Output: YES @@ -194,12 +194,12 @@ LOW PRIORITY SIGNALS: Examples: # Word repetition but complete -[{"role": "assistant", "content": "I can help with that."}, +[{"role": "assistant", "content": "I can help with that."}, {"role": "user", "content": "What what is the time right now"}] Output: YES # Missing punctuation but complete -[{"role": "assistant", "content": "I can explain that."}, +[{"role": "assistant", "content": "I can explain that."}, {"role": "user", "content": "Please tell me how computers work"}] Output: YES @@ -211,12 +211,12 @@ Output: YES Examples: # Filler words but complete -[{"role": "assistant", "content": "What would you like to know?"}, +[{"role": "assistant", "content": "What would you like to know?"}, {"role": "user", "content": "Um uh how do airplanes fly"}] Output: YES # Thinking pause but incomplete -[{"role": "assistant", "content": "I can explain anything."}, +[{"role": "assistant", "content": "I can explain anything."}, {"role": "user", "content": "Well um I want to know about the"}] Output: NO @@ -241,17 +241,17 @@ DECISION RULES: Examples: # Incomplete despite corrections -[{"role": "assistant", "content": "What would you like to know about?"}, +[{"role": "assistant", "content": "What would you like to know about?"}, {"role": "user", "content": "Can you tell me about"}] Output: NO # Complete despite multiple artifacts -[{"role": "assistant", "content": "I can help you learn."}, +[{"role": "assistant", "content": "I can help you learn."}, {"role": "user", "content": "How do you I mean what's the best way to learn programming"}] Output: YES # Trailing off incomplete -[{"role": "assistant", "content": "I can explain anything."}, +[{"role": "assistant", "content": "I can explain anything."}, {"role": "user", "content": "I was wondering if you could tell me why"}] Output: NO """ @@ -268,6 +268,7 @@ class StatementJudgeContextFilter(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) # We must not block system frames. if isinstance(frame, SystemFrame): await self.push_frame(frame, direction) @@ -319,6 +320,8 @@ class CompletenessCheck(FrameProcessor): self._notifier = notifier async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame) and frame.text == "YES": logger.debug("!!! Completeness check YES") await self.push_frame(UserStoppedSpeakingFrame()) @@ -341,6 +344,8 @@ class OutputGate(FrameProcessor): self._gate_open = True async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # We must not block system frames. if isinstance(frame, SystemFrame): if isinstance(frame, StartFrame): diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index e506372a5..1ff8aa23e 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -90,6 +90,8 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): self._user_speaking = False async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # ignore context frame if isinstance(frame, OpenAILLMContextFrame): return @@ -131,6 +133,8 @@ class CompletenessCheck(FrameProcessor): self._audio_accumulator = audio_accumulator async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame) and frame.text.startswith("YES"): logger.debug("Completeness check YES") await self.push_frame(UserStoppedSpeakingFrame()) @@ -155,6 +159,8 @@ class OutputGate(FrameProcessor): self._gate_open = True async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # We must not block system frames. if isinstance(frame, SystemFrame): if isinstance(frame, StartFrame): diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index abeb62043..843d24e1f 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -95,6 +95,8 @@ class UserAudioCollector(FrameProcessor): self._user_speaking = False async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if isinstance(frame, TranscriptionFrame): # We could gracefully handle both audio input and text/transcription input ... # but let's leave that as an exercise to the reader. :-) @@ -133,6 +135,8 @@ class InputTranscriptionContextFilter(FrameProcessor): """ async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if isinstance(frame, SystemFrame): # We don't want to block system frames. await self.push_frame(frame, direction) @@ -206,6 +210,8 @@ class InputTranscriptionFrameEmitter(FrameProcessor): self._aggregation = "" async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): self._aggregation += frame.text elif isinstance(frame, LLMFullResponseEndFrame): @@ -256,6 +262,8 @@ class TranscriptionContextFixup(FrameProcessor): audio_part.text = self._transcript async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if isinstance(frame, LLMDemoTranscriptionFrame): logger.info(f"Transcription from Gemini: {frame.text}") self._transcript = frame.text diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 1c412e88a..54c2013b4 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -81,6 +81,8 @@ class TalkingAnimation(FrameProcessor): self._is_talking = False async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, BotStartedSpeakingFrame): if not self._is_talking: await self.push_frame(talking_frame) @@ -101,6 +103,8 @@ class UserImageRequester(FrameProcessor): self.participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if self.participant_id and isinstance(frame, TextFrame): if frame.text == user_request_answer: await self.push_frame( @@ -117,6 +121,8 @@ class TextFilterProcessor(FrameProcessor): self.text = text async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): if frame.text != self.text: await self.push_frame(frame) @@ -126,6 +132,8 @@ class TextFilterProcessor(FrameProcessor): class ImageFilterProcessor(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if not isinstance(frame, ImageRawFrame): await self.push_frame(frame, direction) diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index 0ce46e50e..991df1cd1 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -95,6 +95,8 @@ class TalkingAnimation(FrameProcessor): frame: The incoming frame to process direction: The direction of frame flow in the pipeline """ + await super().process_frame(frame, direction) + # Switch to talking animation when bot starts speaking if isinstance(frame, BotStartedSpeakingFrame): if not self._is_talking: diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index 02685a99b..a3a68c839 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -95,6 +95,8 @@ class TalkingAnimation(FrameProcessor): frame: The incoming frame to process direction: The direction of frame flow in the pipeline """ + await super().process_frame(frame, direction) + # Switch to talking animation when bot starts speaking if isinstance(frame, BotStartedSpeakingFrame): if not self._is_talking: diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/src/processors.py index dd46f9c82..6aa9ad7ab 100644 --- a/examples/storytelling-chatbot/src/processors.py +++ b/examples/storytelling-chatbot/src/processors.py @@ -54,6 +54,8 @@ class StoryImageProcessor(FrameProcessor): self._fal_service = fal_service async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, StoryImageFrame): try: async with timeout(7): @@ -88,6 +90,8 @@ class StoryProcessor(FrameProcessor): self._story = story async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, UserStoppedSpeakingFrame): # Send an app message to the UI await self.push_frame(DailyTransportMessageFrame(CUE_ASSISTANT_TURN)) diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index 59f495360..e654c0159 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -51,6 +51,8 @@ class TranslationProcessor(FrameProcessor): self._language = language async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): context = [ { @@ -76,6 +78,8 @@ class TranslationSubtitles(FrameProcessor): # subtitles. # async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): message = {"language": self._language, "text": frame.text} await self.push_frame(DailyTransportMessageFrame(message)) diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 7499192fb..40bfea90d 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -28,6 +28,8 @@ class Source(FrameProcessor): self._push_frame_func = push_frame_func async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + match direction: case FrameDirection.UPSTREAM: if isinstance(frame, SystemFrame): @@ -49,6 +51,8 @@ class Sink(FrameProcessor): self._push_frame_func = push_frame_func async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + match direction: case FrameDirection.UPSTREAM: await self.push_frame(frame, direction) @@ -116,6 +120,8 @@ class ParallelPipeline(BasePipeline): self._down_task = loop.create_task(self._process_down_queue()) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, StartFrame): await self._start_tasks() diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py index 457b70cab..703f911fe 100644 --- a/src/pipecat/pipeline/pipeline.py +++ b/src/pipecat/pipeline/pipeline.py @@ -17,6 +17,8 @@ class PipelineSource(FrameProcessor): self._upstream_push_frame = upstream_push_frame async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + match direction: case FrameDirection.UPSTREAM: await self._upstream_push_frame(frame, direction) @@ -30,6 +32,8 @@ class PipelineSink(FrameProcessor): self._downstream_push_frame = downstream_push_frame async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + match direction: case FrameDirection.UPSTREAM: await self.push_frame(frame, direction) @@ -70,6 +74,8 @@ class Pipeline(BasePipeline): await self._cleanup_processors() async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if direction == FrameDirection.DOWNSTREAM: await self._source.queue_frame(frame, FrameDirection.DOWNSTREAM) elif direction == FrameDirection.UPSTREAM: diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py index 4dcf190de..20f4275e4 100644 --- a/src/pipecat/pipeline/sync_parallel_pipeline.py +++ b/src/pipecat/pipeline/sync_parallel_pipeline.py @@ -31,6 +31,8 @@ class Source(FrameProcessor): self._up_queue = upstream_queue async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + match direction: case FrameDirection.UPSTREAM: await self._up_queue.put(frame) @@ -44,6 +46,8 @@ class Sink(FrameProcessor): self._down_queue = downstream_queue async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + match direction: case FrameDirection.UPSTREAM: await self.push_frame(frame, direction) @@ -99,6 +103,8 @@ class SyncParallelPipeline(BasePipeline): # async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # The last processor of each pipeline needs to be synchronous otherwise # this element won't work. Since, we know it should be synchronous we # push a SyncFrame. Since frames are ordered we know this frame will be diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index d8bada663..f09013a58 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -45,6 +45,8 @@ class Source(FrameProcessor): self._up_queue = up_queue async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + match direction: case FrameDirection.UPSTREAM: await self._handle_upstream_frame(frame) @@ -73,6 +75,8 @@ class Sink(FrameProcessor): self._down_queue = down_queue async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # We really just want to know when the EndFrame reached the sink. if isinstance(frame, EndFrame): await self._down_queue.put(frame) diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index 763dc456c..c39a35c82 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -56,6 +56,8 @@ class GatedAggregator(FrameProcessor): self._accumulator: List[Tuple[Frame, FrameDirection]] = [] async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # We must not block system frames. if isinstance(frame, SystemFrame): await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index 9b0d77d32..71a540dd4 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -24,6 +24,8 @@ class GatedOpenAILLMContextAggregator(FrameProcessor): self._last_context_frame = None async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, StartFrame): await self.push_frame(frame) await self._start() diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 544b49dda..479746471 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -86,6 +86,8 @@ class LLMResponseAggregator(FrameProcessor): # and T2 would be dropped. async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + send_aggregation = False if isinstance(frame, self._start_frame): @@ -238,6 +240,8 @@ class LLMFullResponseAggregator(FrameProcessor): self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): self._aggregation += frame.text elif isinstance(frame, LLMFullResponseEndFrame): diff --git a/src/pipecat/processors/aggregators/sentence.py b/src/pipecat/processors/aggregators/sentence.py index ab400b2a0..d0c593a83 100644 --- a/src/pipecat/processors/aggregators/sentence.py +++ b/src/pipecat/processors/aggregators/sentence.py @@ -33,6 +33,8 @@ class SentenceAggregator(FrameProcessor): self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # We ignore interim description at this point. if isinstance(frame, InterimTranscriptionFrame): return diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index 78287127f..903019059 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -85,6 +85,8 @@ class ResponseAggregator(FrameProcessor): # and T2 would be dropped. async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + send_aggregation = False if isinstance(frame, self._start_frame): diff --git a/src/pipecat/processors/aggregators/vision_image_frame.py b/src/pipecat/processors/aggregators/vision_image_frame.py index 3a4eda330..d07337f06 100644 --- a/src/pipecat/processors/aggregators/vision_image_frame.py +++ b/src/pipecat/processors/aggregators/vision_image_frame.py @@ -31,6 +31,8 @@ class VisionImageFrameAggregator(FrameProcessor): self._describe_text = None async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): self._describe_text = frame.text elif isinstance(frame, InputImageRawFrame): diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py index 356ef4388..4f9bc85d0 100644 --- a/src/pipecat/processors/async_generator.py +++ b/src/pipecat/processors/async_generator.py @@ -24,6 +24,8 @@ class AsyncGeneratorProcessor(FrameProcessor): self._data_queue = asyncio.Queue() async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, (CancelFrame, EndFrame)): diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index c7bb36736..488a251f0 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -68,6 +68,8 @@ class AudioBufferProcessor(FrameProcessor): self._bot_audio_buffer = bytearray() async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # Include all audio from the user. if isinstance(frame, InputAudioRawFrame): resampled = resample_audio(frame.audio, frame.sample_rate, self._sample_rate) diff --git a/src/pipecat/processors/audio/vad/silero.py b/src/pipecat/processors/audio/vad/silero.py index 1db510f24..4aa32a163 100644 --- a/src/pipecat/processors/audio/vad/silero.py +++ b/src/pipecat/processors/audio/vad/silero.py @@ -39,6 +39,8 @@ class SileroVAD(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, AudioRawFrame): await self._analyze_audio(frame) if self._audio_passthrough: diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py index e87034a1a..11f2e601a 100644 --- a/src/pipecat/processors/filters/frame_filter.py +++ b/src/pipecat/processors/filters/frame_filter.py @@ -26,5 +26,7 @@ class FrameFilter(FrameProcessor): return isinstance(frame, ControlFrame) or isinstance(frame, SystemFrame) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if self._should_passthrough_frame(frame): await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index 522a89324..e38cea3e0 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -29,6 +29,8 @@ class FunctionFilter(FrameProcessor): return isinstance(frame, SystemFrame) or direction != self._direction async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + passthrough = self._should_passthrough_frame(frame, direction) allowed = await self._filter(frame) if passthrough or allowed: diff --git a/src/pipecat/processors/filters/identity_filter.py b/src/pipecat/processors/filters/identity_filter.py index c837e02a7..d6f896b73 100644 --- a/src/pipecat/processors/filters/identity_filter.py +++ b/src/pipecat/processors/filters/identity_filter.py @@ -26,4 +26,5 @@ class IdentityFilter(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): """Process an incoming frame by passing it through unchanged.""" + await super().process_frame(frame, direction) await self.push_frame(frame, direction) diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index 441f32fb9..f1a7afbef 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -45,6 +45,8 @@ class WakeCheckFilter(FrameProcessor): self._wake_patterns.append(pattern) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + try: if isinstance(frame, TranscriptionFrame): p = self._participant_states.get(frame.user_id) diff --git a/src/pipecat/processors/filters/wake_notifier_filter.py b/src/pipecat/processors/filters/wake_notifier_filter.py index 7623b6da8..a7f074ccb 100644 --- a/src/pipecat/processors/filters/wake_notifier_filter.py +++ b/src/pipecat/processors/filters/wake_notifier_filter.py @@ -32,6 +32,8 @@ class WakeNotifierFilter(FrameProcessor): self._filter = filter async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, self._types) and await self._filter(frame): await self._notifier.notify() diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index e0806a0bf..52066b4f4 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -161,13 +161,6 @@ class FrameProcessor: def get_clock(self) -> BaseClock: return self._clock - async def pause_processing_frames(self): - self.__should_block_frames = True - - async def resume_processing_frames(self): - self.__input_event.set() - self.__should_block_frames = False - async def queue_frame( self, frame: Frame, @@ -182,13 +175,32 @@ class FrameProcessor: if isinstance(frame, SystemFrame): # We don't want to queue system frames. - await self._process_frame(frame, direction) + await self.process_frame(frame, direction) else: # We queue everything else. await self.__input_queue.put((frame, direction, callback)) + async def pause_processing_frames(self): + self.__should_block_frames = True + + async def resume_processing_frames(self): + self.__input_event.set() + self.__should_block_frames = False + async def process_frame(self, frame: Frame, direction: FrameDirection): - pass + if isinstance(frame, StartFrame): + self._clock = frame.clock + self._allow_interruptions = frame.allow_interruptions + self._enable_metrics = frame.enable_metrics + self._enable_usage_metrics = frame.enable_usage_metrics + self._report_only_initial_ttfb = frame.report_only_initial_ttfb + elif isinstance(frame, StartInterruptionFrame): + await self._start_interruption() + await self.stop_all_metrics() + elif isinstance(frame, StopInterruptionFrame): + self._should_report_ttfb = True + elif isinstance(frame, CancelFrame): + self._cancelling = True async def push_error(self, error: ErrorFrame): await self.push_frame(error, FrameDirection.UPSTREAM) @@ -216,28 +228,6 @@ class FrameProcessor: raise Exception(f"Event handler {event_name} already registered") self._event_handlers[event_name] = [] - # - # Frame processing - # - - async def _process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, StartFrame): - self._clock = frame.clock - self._allow_interruptions = frame.allow_interruptions - self._enable_metrics = frame.enable_metrics - self._enable_usage_metrics = frame.enable_usage_metrics - self._report_only_initial_ttfb = frame.report_only_initial_ttfb - elif isinstance(frame, StartInterruptionFrame): - await self._start_interruption() - await self.stop_all_metrics() - elif isinstance(frame, StopInterruptionFrame): - self._should_report_ttfb = True - elif isinstance(frame, CancelFrame): - self._cancelling = True - - # Call subclass. - await self.process_frame(frame, direction) - # # Handle interruptions # @@ -299,7 +289,7 @@ class FrameProcessor: (frame, direction, callback) = await self.__input_queue.get() # Process the frame. - await self._process_frame(frame, direction) + await self.process_frame(frame, direction) # If this frame has an associated callback, call it now. if callback: diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index 25de11070..c0b657244 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -36,6 +36,8 @@ class LangchainProcessor(FrameProcessor): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, LLMMessagesFrame): # Messages are accumulated on the context as a list of messages. # The last one by the human is the one we want to send to the LLM. diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index b91abb181..471bdbb88 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -380,6 +380,8 @@ class RTVISpeakingProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, (UserStartedSpeakingFrame, UserStoppedSpeakingFrame)): @@ -413,6 +415,8 @@ class RTVIUserTranscriptionProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, (TranscriptionFrame, InterimTranscriptionFrame)): @@ -442,6 +446,8 @@ class RTVIUserLLMTextProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, OpenAILLMContextFrame): @@ -467,6 +473,8 @@ class RTVIBotTranscriptionProcessor(RTVIFrameProcessor): self._aggregation = "" async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, UserStartedSpeakingFrame): @@ -488,6 +496,8 @@ class RTVIBotLLMProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, LLMFullResponseStartFrame): @@ -504,6 +514,8 @@ class RTVIBotTTSProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, TTSStartedFrame): @@ -520,6 +532,8 @@ class RTVIMetricsProcessor(RTVIFrameProcessor): super().__init__(**kwargs) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) if isinstance(frame, MetricsFrame): @@ -628,6 +642,8 @@ class RTVIProcessor(FrameProcessor): await self._push_transport_message(message, exclude_none=False) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # Specific system frames if isinstance(frame, StartFrame): # Push StartFrame before start(), because we want StartFrame to be diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 09456f12e..649a2c529 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -66,6 +66,8 @@ class GStreamerPipelineSource(FrameProcessor): bus.connect("message", self._on_gstreamer_message) async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # Specific system frames if isinstance(frame, StartFrame): # Push StartFrame before start(), because we want StartFrame to be diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index 80902ee59..e674b6b84 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -35,6 +35,8 @@ class IdleFrameProcessor(FrameProcessor): self._create_idle_task() async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + await self.push_frame(frame, direction) # If we are not waiting for any specific frame set the event, otherwise diff --git a/src/pipecat/processors/text_transformer.py b/src/pipecat/processors/text_transformer.py index 90ef6b8bc..79e9b885e 100644 --- a/src/pipecat/processors/text_transformer.py +++ b/src/pipecat/processors/text_transformer.py @@ -27,6 +27,8 @@ class StatelessTextTransformer(FrameProcessor): self._transform_fn = transform_fn async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, TextFrame): result = self._transform_fn(frame.text) if isinstance(result, Coroutine): diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 91cbd2334..160c49908 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -43,6 +43,8 @@ class UserIdleProcessor(FrameProcessor): await self._idle_task async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # Check for end frames before processing if isinstance(frame, (EndFrame, CancelFrame)): await self._stop() diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index e324d413c..e0f16e220 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -110,6 +110,8 @@ class AIService(FrameProcessor): logger.warning(f"Unknown setting for {self.name} service: {key}") async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, StartFrame): await self.start(frame) elif isinstance(frame, CancelFrame): diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index e61fb394c..bfae861dc 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -92,6 +92,7 @@ class SimliVideoService(FrameProcessor): pass async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) if isinstance(frame, StartFrame): await self.push_frame(frame, direction) await self._start_connection() diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 64583c758..025a5bed2 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -79,6 +79,8 @@ class BaseInputTransport(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # Specific system frames if isinstance(frame, StartFrame): # Push StartFrame before start(), because we want StartFrame to be diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 56c86076a..573b67717 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -120,6 +120,8 @@ class BaseOutputTransport(FrameProcessor): # async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # # System frames (like StartInterruptionFrame) are pushed # immediately. Other frames require order so they are put in the sink diff --git a/src/pipecat/utils/test_frame_processor.py b/src/pipecat/utils/test_frame_processor.py index ec37efd80..e46bae7ad 100644 --- a/src/pipecat/utils/test_frame_processor.py +++ b/src/pipecat/utils/test_frame_processor.py @@ -13,6 +13,8 @@ class TestFrameProcessor(FrameProcessor): super().__init__() async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if not self.test_frames[ 0 ]: # then we've run out of required frames but the generator is still going? diff --git a/tests/test_langchain.py b/tests/test_langchain.py index b1f8f618d..d30d213bd 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -42,6 +42,8 @@ class TestLangchain(unittest.IsolatedAsyncioTestCase): return self.name async def process_frame(self, frame, direction): + await super().process_frame(frame, direction) + if isinstance(frame, LLMFullResponseStartFrame): self.start_collecting = True elif isinstance(frame, TextFrame) and self.start_collecting: From 752720b4d50849f0afe8374cc07bd8237dda399e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 12 Dec 2024 17:25:38 -0800 Subject: [PATCH 15/90] AIService: add missing super().process_frame() --- src/pipecat/services/ai_services.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index e324d413c..e0f16e220 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -110,6 +110,8 @@ class AIService(FrameProcessor): logger.warning(f"Unknown setting for {self.name} service: {key}") async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if isinstance(frame, StartFrame): await self.start(frame) elif isinstance(frame, CancelFrame): From 337d42133885d55194493b76436aa249a05f0b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 12 Dec 2024 17:40:01 -0800 Subject: [PATCH 16/90] transports: disconnect client first --- pyproject.toml | 2 +- .../transports/network/websocket_server.py | 6 ++-- src/pipecat/transports/services/daily.py | 26 ++++++++-------- src/pipecat/transports/services/livekit.py | 31 +++---------------- 4 files changed, 22 insertions(+), 43 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f8b122580..fd0c1be56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.57.2" ] krisp = [ "pipecat-ai-krisp~=0.3.0" ] langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] -livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1", "tenacity~=8.5.0" ] +livekit = [ "livekit~=0.12.5", "livekit-api~=0.7.1", "tenacity~=8.5.0" ] lmnt = [ "lmnt~=1.1.4" ] local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.8", "transformers~=4.44.0" ] diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 711bc7596..ce9b9614d 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -69,18 +69,18 @@ class WebsocketServerInputTransport(BaseInputTransport): self._stop_server_event = asyncio.Event() async def start(self, frame: StartFrame): - self._server_task = self.get_event_loop().create_task(self._server_task_handler()) await super().start(frame) + self._server_task = self.get_event_loop().create_task(self._server_task_handler()) async def stop(self, frame: EndFrame): - await super().stop(frame) self._stop_server_event.set() await self._server_task + await super().stop(frame) async def cancel(self, frame: CancelFrame): - await super().cancel(frame) self._stop_server_event.set() await self._server_task + await super().cancel(frame) async def _server_task_handler(self): logger.info(f"Starting websocket server on {self._host}:{self._port}") diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 7456ef816..44c9f37ee 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -694,19 +694,17 @@ class DailyInputTransport(BaseInputTransport): self._audio_in_task = self.get_event_loop().create_task(self._audio_in_task_handler()) async def stop(self, frame: EndFrame): + # Leave the room. + await self._client.leave() + # Stop audio thread. + if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): + self._audio_in_task.cancel() + await self._audio_in_task + self._audio_in_task = None # Parent stop. await super().stop(frame) - # Leave the room. - await self._client.leave() - # Stop audio thread. - if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): - self._audio_in_task.cancel() - await self._audio_in_task - self._audio_in_task = None async def cancel(self, frame: CancelFrame): - # Parent stop. - await super().cancel(frame) # Leave the room. await self._client.leave() # Stop audio thread. @@ -714,6 +712,8 @@ class DailyInputTransport(BaseInputTransport): self._audio_in_task.cancel() await self._audio_in_task self._audio_in_task = None + # Parent stop. + await super().cancel(frame) async def cleanup(self): await super().cleanup() @@ -817,16 +817,16 @@ class DailyOutputTransport(BaseOutputTransport): await self._client.join() async def stop(self, frame: EndFrame): + # Leave the room. + await self._client.leave() # Parent stop. await super().stop(frame) - # Leave the room. - await self._client.leave() async def cancel(self, frame: CancelFrame): - # Parent stop. - await super().cancel(frame) # Leave the room. await self._client.leave() + # Parent stop. + await super().cancel(frame) async def cleanup(self): await super().cleanup() diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index f53c8332f..ded16182f 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -324,28 +324,19 @@ class LiveKitInputTransport(BaseInputTransport): logger.info("LiveKitInputTransport started") async def stop(self, frame: EndFrame): + await self._client.disconnect() if self._audio_in_task: self._audio_in_task.cancel() - try: - await self._audio_in_task - except asyncio.CancelledError: - pass + await self._audio_in_task await super().stop(frame) - await self._client.disconnect() logger.info("LiveKitInputTransport stopped") - async def process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, EndFrame): - await self.stop(frame) - else: - await super().process_frame(frame, direction) - async def cancel(self, frame: CancelFrame): - await super().cancel(frame) await self._client.disconnect() if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): self._audio_in_task.cancel() await self._audio_in_task + await super().cancel(frame) def vad_analyzer(self) -> VADAnalyzer | None: return self._vad_analyzer @@ -407,19 +398,13 @@ class LiveKitOutputTransport(BaseOutputTransport): logger.info("LiveKitOutputTransport started") async def stop(self, frame: EndFrame): - await super().stop(frame) await self._client.disconnect() + await super().stop(frame) logger.info("LiveKitOutputTransport stopped") - async def process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, EndFrame): - await self.stop(frame) - else: - await super().process_frame(frame, direction) - async def cancel(self, frame: CancelFrame): - await super().cancel(frame) await self._client.disconnect() + await super().cancel(frame) async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)): @@ -526,12 +511,6 @@ class LiveKitTransport(BaseTransport): async def _on_disconnected(self): await self._call_event_handler("on_disconnected") - # Attempt to reconnect - try: - await self._client.connect() - await self._call_event_handler("on_connected") - except Exception as e: - logger.error(f"Failed to reconnect: {e}") async def _on_participant_connected(self, participant_id: str): await self._call_event_handler("on_participant_connected", participant_id) From ccc96994e9bd195029b0f6012ee48a35598befb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 12 Dec 2024 19:09:36 -0800 Subject: [PATCH 17/90] pyproject: update livekit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fd0c1be56..862981c7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.57.2" ] krisp = [ "pipecat-ai-krisp~=0.3.0" ] langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] -livekit = [ "livekit~=0.12.5", "livekit-api~=0.7.1", "tenacity~=8.5.0" ] +livekit = [ "livekit~=0.18.2", "livekit-api~=0.8.0", "tenacity~=8.5.0" ] lmnt = [ "lmnt~=1.1.4" ] local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.8", "transformers~=4.44.0" ] From 420ce16807eb5ac21946e2b2dfa0a819f45b1344 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 12 Dec 2024 22:15:44 -0800 Subject: [PATCH 18/90] riva: fix FastPitchTTSService audio stuttering --- CHANGELOG.md | 2 ++ src/pipecat/services/riva.py | 21 +++++++++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca41b8a4..22e08ef4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an audio stuttering issue in `FastPitchTTSService`. + - Fixed a `BaseOutputTransport` issue that was causing non-audio frames being processed before the previous audio frames were played. This will allow, for example, sending a frame `A` after a `TTSSpeakFrame` and the frame `A` will diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index 6be722d49..470fe6dc9 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -76,7 +76,10 @@ class FastPitchTTSService(TTSService): ) async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: - def read_audio_responses(): + def read_audio_responses(queue: asyncio.Queue): + def add_response(r): + asyncio.run_coroutine_threadsafe(queue.put(r), self.get_event_loop()) + try: responses = self._service.synthesize_online( text, @@ -87,26 +90,32 @@ class FastPitchTTSService(TTSService): quality=self._quality, custom_dictionary={}, ) - return responses + for r in responses: + add_response(r) + add_response(None) except Exception as e: logger.error(f"{self} exception: {e}") - return [] + add_response(None) await self.start_ttfb_metrics() yield TTSStartedFrame() logger.debug(f"Generating TTS: [{text}]") - responses = await asyncio.to_thread(read_audio_responses) - for resp in responses: + queue = asyncio.Queue() + await asyncio.to_thread(read_audio_responses, queue) + + # Wait for the thread to start. + resp = await queue.get() + while resp: await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame( audio=resp.audio, sample_rate=self._sample_rate, num_channels=1, ) yield frame + resp = await queue.get() await self.start_tts_usage_metrics(text) yield TTSStoppedFrame() From aac907aadb2b819f853a2b465b03c789e3265769 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 13 Dec 2024 07:22:05 -0800 Subject: [PATCH 19/90] riva: make sure we don't block on fastpitch --- src/pipecat/services/riva.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index 470fe6dc9..8ab0e99d8 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -35,6 +35,8 @@ except ModuleNotFoundError as e: ) raise Exception(f"Missing module: {e}") +FASTPITCH_TIMEOUT_SECS = 5 + class FastPitchTTSService(TTSService): class InputParams(BaseModel): @@ -102,20 +104,23 @@ class FastPitchTTSService(TTSService): logger.debug(f"Generating TTS: [{text}]") - queue = asyncio.Queue() - await asyncio.to_thread(read_audio_responses, queue) + try: + queue = asyncio.Queue() + await asyncio.to_thread(read_audio_responses, queue) - # Wait for the thread to start. - resp = await queue.get() - while resp: - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame( - audio=resp.audio, - sample_rate=self._sample_rate, - num_channels=1, - ) - yield frame - resp = await queue.get() + # Wait for the thread to start. + resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS) + while resp: + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame( + audio=resp.audio, + sample_rate=self._sample_rate, + num_channels=1, + ) + yield frame + resp = await asyncio.wait_for(queue.get(), FASTPITCH_TIMEOUT_SECS) + except asyncio.TimeoutError: + logger.error(f"{self} timeout waiting for audio response") await self.start_tts_usage_metrics(text) yield TTSStoppedFrame() From 16d7fb2c4a5a1dd79ed464350b72d5167c5cc11e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 22:50:39 -0500 Subject: [PATCH 20/90] Remove default 5 min exp time for created rooms, add docstrings --- CHANGELOG.md | 6 + .../transports/services/helpers/daily_rest.py | 139 +++++++++++++++++- 2 files changed, 137 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca41b8a4..dc9970484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa). +### Changed + +- Changed: Room expiration (`exp`) in `DailyRoomProperties` is now optional + (None) by default instead of automatically setting a 5-minute expiration + time. You must explicitly set expiration time if desired. + ### Deprecated - `AWSTTSService` is now deprecated, use `PollyTTSService` instead. diff --git a/src/pipecat/transports/services/helpers/daily_rest.py b/src/pipecat/transports/services/helpers/daily_rest.py index 4f15fc28a..f2b7c7e59 100644 --- a/src/pipecat/transports/services/helpers/daily_rest.py +++ b/src/pipecat/transports/services/helpers/daily_rest.py @@ -4,23 +4,29 @@ # SPDX-License-Identifier: BSD 2-Clause License # -""" -Daily REST Helpers +"""Daily REST Helpers. Methods that wrap the Daily API to create rooms, check room URLs, and get meeting tokens. - """ -import aiohttp import time - +from typing import Literal, Optional from urllib.parse import urlparse -from pydantic import Field, BaseModel, ValidationError -from typing import Literal, Optional +import aiohttp +from pydantic import BaseModel, Field, ValidationError class DailyRoomSipParams(BaseModel): + """SIP configuration parameters for Daily rooms. + + Attributes: + display_name: Name shown for the SIP endpoint + video: Whether video is enabled for SIP + sip_mode: SIP connection mode, typically 'dial-in' + num_endpoints: Number of allowed SIP endpoints + """ + display_name: str = "sw-sip-dialin" video: bool = False sip_mode: str = "dial-in" @@ -28,7 +34,19 @@ class DailyRoomSipParams(BaseModel): class DailyRoomProperties(BaseModel, extra="allow"): - exp: float = Field(default_factory=lambda: time.time() + 5 * 60) + """Properties for configuring a Daily room. + + Attributes: + exp: Optional Unix epoch timestamp for room expiration (e.g., time.time() + 300 for 5 minutes) + enable_chat: Whether chat is enabled in the room + enable_emoji_reactions: Whether emoji reactions are enabled + eject_at_room_exp: Whether to remove participants when room expires + enable_dialout: Whether SIP dial-out is enabled + sip: SIP configuration parameters + sip_uri: SIP URI information returned by Daily + """ + + exp: Optional[float] = None enable_chat: bool = False enable_emoji_reactions: bool = False eject_at_room_exp: bool = True @@ -38,6 +56,11 @@ class DailyRoomProperties(BaseModel, extra="allow"): @property def sip_endpoint(self) -> str: + """Get the SIP endpoint URI if available. + + Returns: + str: SIP endpoint URI or empty string if not available + """ if not self.sip_uri: return "" else: @@ -45,12 +68,32 @@ class DailyRoomProperties(BaseModel, extra="allow"): class DailyRoomParams(BaseModel): + """Parameters for creating a Daily room. + + Attributes: + name: Optional custom name for the room + privacy: Room privacy setting ('private' or 'public') + properties: Room configuration properties + """ + name: Optional[str] = None privacy: Literal["private", "public"] = "public" properties: DailyRoomProperties = Field(default_factory=DailyRoomProperties) class DailyRoomObject(BaseModel): + """Represents a Daily room returned by the API. + + Attributes: + id: Unique room identifier + name: Room name + api_created: Whether room was created via API + privacy: Room privacy setting ('private' or 'public') + url: Full URL for joining the room + created_at: Timestamp of room creation in ISO 8601 format (e.g., "2019-01-26T09:01:22.000Z"). + config: Room configuration properties + """ + id: str name: str api_created: bool @@ -61,6 +104,16 @@ class DailyRoomObject(BaseModel): class DailyRESTHelper: + """Helper class for interacting with Daily's REST API. + + Provides methods for creating, managing, and accessing Daily rooms. + + Args: + daily_api_key: Your Daily API key + daily_api_url: Daily API base URL (e.g. "https://api.daily.co/v1") + aiohttp_session: Async HTTP session for making requests + """ + def __init__( self, *, @@ -73,13 +126,40 @@ class DailyRESTHelper: self.aiohttp_session = aiohttp_session def get_name_from_url(self, room_url: str) -> str: + """Extract room name from a Daily room URL. + + Args: + room_url: Full Daily room URL + + Returns: + str: Room name portion of the URL + """ return urlparse(room_url).path[1:] async def get_room_from_url(self, room_url: str) -> DailyRoomObject: + """Get room details from a Daily room URL. + + Args: + room_url: Full Daily room URL + + Returns: + DailyRoomObject: DailyRoomObject instance for the room + """ room_name = self.get_name_from_url(room_url) return await self._get_room_from_name(room_name) async def create_room(self, params: DailyRoomParams) -> DailyRoomObject: + """Create a new Daily room. + + Args: + params: Room configuration parameters + + Returns: + DailyRoomObject: DailyRoomObject instance for the created room + + Raises: + Exception: If room creation fails or response is invalid + """ headers = {"Authorization": f"Bearer {self.daily_api_key}"} json = {**params.model_dump(exclude_none=True)} async with self.aiohttp_session.post( @@ -101,6 +181,19 @@ class DailyRESTHelper: async def get_token( self, room_url: str, expiry_time: float = 60 * 60, owner: bool = True ) -> str: + """Generate a meeting token for user to join a Daily room. + + Args: + room_url: Daily room URL + expiry_time: Token validity duration in seconds (default: 1 hour) + owner: Whether token has owner privileges + + Returns: + str: Meeting token + + Raises: + Exception: If token generation fails or room URL is missing + """ if not room_url: raise Exception( "No Daily room specified. You must specify a Daily room in order a token to be generated." @@ -124,10 +217,29 @@ class DailyRESTHelper: return data["token"] async def delete_room_by_url(self, room_url: str) -> bool: + """Delete a room using its URL. + + Args: + room_url: Daily room URL + + Returns: + bool: True if deletion was successful + """ room_name = self.get_name_from_url(room_url) return await self.delete_room_by_name(room_name) async def delete_room_by_name(self, room_name: str) -> bool: + """Delete a room using its name. + + Args: + room_name: Name of the room to delete + + Returns: + bool: True if deletion was successful + + Raises: + Exception: If deletion fails (excluding 404 Not Found) + """ headers = {"Authorization": f"Bearer {self.daily_api_key}"} async with self.aiohttp_session.delete( f"{self.daily_api_url}/rooms/{room_name}", headers=headers @@ -139,6 +251,17 @@ class DailyRESTHelper: return True async def _get_room_from_name(self, room_name: str) -> DailyRoomObject: + """Internal method to get room details by name. + + Args: + room_name: Name of the room + + Returns: + DailyRoomObject: DailyRoomObject instance for the room + + Raises: + Exception: If room is not found or response is invalid + """ headers = {"Authorization": f"Bearer {self.daily_api_key}"} async with self.aiohttp_session.get( f"{self.daily_api_url}/rooms/{room_name}", headers=headers From f90cbe8086af56bbca29d439491769aabe1e1c0e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sun, 15 Dec 2024 14:30:20 -0500 Subject: [PATCH 21/90] Fix a bunch of README docs links --- README.md | 24 +++++++++---------- examples/simple-chatbot/README.md | 4 ++-- .../examples/javascript/README.md | 2 +- .../simple-chatbot/examples/react/README.md | 2 +- examples/studypal/env.example | 2 +- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 86e3ddd3e..f63d85167 100644 --- a/README.md +++ b/README.md @@ -55,19 +55,19 @@ pip install "pipecat-ai[option,...]" Available options include: -| Category | Services | Install Command Example | -| ------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/api-reference/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/api-reference/services/stt/azure), [Deepgram](https://docs.pipecat.ai/api-reference/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/api-reference/services/stt/gladia), [Whisper](https://docs.pipecat.ai/api-reference/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | -| LLMs | [Anthropic](https://docs.pipecat.ai/api-reference/services/llm/anthropic), [Azure](https://docs.pipecat.ai/api-reference/services/llm/azure), [Fireworks AI](https://docs.pipecat.ai/api-reference/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/api-reference/services/llm/gemini), [Grok](https://docs.pipecat.ai/api-reference/services/llm/grok), [Groq](https://docs.pipecat.ai/api-reference/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/api-reference/services/llm/nim), [Ollama](https://docs.pipecat.ai/api-reference/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/api-reference/services/llm/openai), [Together AI](https://docs.pipecat.ai/api-reference/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/api-reference/services/tts/aws), [Azure](https://docs.pipecat.ai/api-reference/services/tts/azure), [Cartesia](https://docs.pipecat.ai/api-reference/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/api-reference/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/api-reference/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/api-reference/services/tts/google), [LMNT](https://docs.pipecat.ai/api-reference/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/api-reference/services/tts/openai), [PlayHT](https://docs.pipecat.ai/api-reference/services/tts/playht), [Rime](https://docs.pipecat.ai/api-reference/services/tts/rime), [XTTS](https://docs.pipecat.ai/api-reference/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | -| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/api-reference/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/api-reference/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | -| Video | [Tavus](https://docs.pipecat.ai/api-reference/services/video/tavus), [Simli](https://docs.pipecat.ai/api-reference/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | -| Vision & Image | [Moondream](https://docs.pipecat.ai/api-reference/services/vision/moondream), [fal](https://docs.pipecat.ai/api-reference/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/api-reference/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/api-reference/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/api-reference/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | -| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/api-reference/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/api-reference/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | +| Category | Services | Install Command Example | +| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | +| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | +| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | +| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | -📚 [View full services documentation →](https://docs.pipecat.ai/api-reference/services/supported-services) +📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) ## Code examples diff --git a/examples/simple-chatbot/README.md b/examples/simple-chatbot/README.md index e9b06855d..4e23c9fae 100644 --- a/examples/simple-chatbot/README.md +++ b/examples/simple-chatbot/README.md @@ -24,12 +24,12 @@ This repository demonstrates a simple AI chatbot with real-time audio/video inte 2. **JavaScript** - - Basic implementation using [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/reference/js/introduction) + - Basic implementation using [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/js/introduction) - No framework dependencies - Good for learning the fundamentals 3. **React** - - Basic impelmentation using [Pipecat React SDK](https://docs.pipecat.ai/client/reference/react/introduction) + - Basic impelmentation using [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction) - Demonstrates the basic client principles with Pipecat React ## Quick Start diff --git a/examples/simple-chatbot/examples/javascript/README.md b/examples/simple-chatbot/examples/javascript/README.md index 74525c1c1..fdd97ce27 100644 --- a/examples/simple-chatbot/examples/javascript/README.md +++ b/examples/simple-chatbot/examples/javascript/README.md @@ -1,6 +1,6 @@ # JavaScript Implementation -Basic implementation using the [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/reference/js/introduction). +Basic implementation using the [Pipecat JavaScript SDK](https://docs.pipecat.ai/client/js/introduction). ## Setup diff --git a/examples/simple-chatbot/examples/react/README.md b/examples/simple-chatbot/examples/react/README.md index 44775a083..892763d18 100644 --- a/examples/simple-chatbot/examples/react/README.md +++ b/examples/simple-chatbot/examples/react/README.md @@ -1,6 +1,6 @@ # React Implementation -Basic implementation using the [Pipecat React SDK](https://docs.pipecat.ai/client/reference/react/introduction). +Basic implementation using the [Pipecat React SDK](https://docs.pipecat.ai/client/react/introduction). ## Setup diff --git a/examples/studypal/env.example b/examples/studypal/env.example index 69245a3d6..6acef284b 100644 --- a/examples/studypal/env.example +++ b/examples/studypal/env.example @@ -1,4 +1,4 @@ -DAILY_SAMPLE_ROOM_URL= # Follow instructions here and put your https://YOURDOMAIN.daily.co/YOURROOM (Instructions: https://docs.pipecat.ai/quickstart#preparing-your-environment) +DAILY_SAMPLE_ROOM_URL= # Follow instructions here and put your https://YOURDOMAIN.daily.co/YOURROOM (Instructions: https://docs.pipecat.ai/getting-started/installation) DAILY_API_KEY= # Create here: https://dashboard.daily.co/developers OPENAI_API_KEY= # Create here: https://platform.openai.com/docs/overview CARTESIA_API_KEY= # Create here: https://play.cartesia.ai/console From facc280599aef59a1abc290fda022c706330c09f Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Mon, 16 Dec 2024 17:47:50 +0530 Subject: [PATCH 22/90] fixing [#868] bug where deepgram client fails due to langauge --- src/pipecat/services/deepgram.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 6578f2873..48193388d 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -138,6 +138,9 @@ class DeepgramSTTService(STTService): merged_options = default_options if live_options: merged_options = LiveOptions(**{**default_options.to_dict(), **live_options.to_dict()}) + + # deepgram connection requires language to be a string + merged_options.language = merged_options.language.value self._settings = merged_options.to_dict() self._client = DeepgramClient( From 64038442ed06237ea3da7258bce851cabf96921f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 16 Dec 2024 09:23:12 -0500 Subject: [PATCH 23/90] Add python-dotenv to dev-requirements.txt --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-requirements.txt b/dev-requirements.txt index 92b6ec4d3..db2cad4fa 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -6,3 +6,4 @@ pytest~=8.3.2 ruff~=0.6.7 setuptools~=72.2.0 setuptools_scm~=8.1.0 +python-dotenv~=1.0.1 \ No newline at end of file From 16948b251deec1b0696f8e8d465595a459759a60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 16 Dec 2024 14:55:33 -0800 Subject: [PATCH 24/90] services: fix infinite websocket-bases TTS services retries Fixes #871 --- CHANGELOG.md | 7 +++ pyproject.toml | 3 +- src/pipecat/services/cartesia.py | 94 +++++++++++++++++------------- src/pipecat/services/elevenlabs.py | 50 +++++++++++----- src/pipecat/services/lmnt.py | 55 +++++++++++------ src/pipecat/services/playht.py | 63 ++++++++++++-------- 6 files changed, 173 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccf5dd5eb..d0d424b18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 only be pushed downstream after the audio generated from `TTSSpeakFrame` has been spoken. +## [0.0.51] - 2024-12-16 + +### Fixed + +- Fixed an issue in websocket-based TTS services that was causing infinite + reconnections (Cartesia, ElevenLabs, PlayHT and LMNT). + ## [0.0.50] - 2024-12-11 ### Added diff --git a/pyproject.toml b/pyproject.toml index 862981c7b..9ee258ff6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "pydantic~=2.8.2", "pyloudnorm~=0.1.1", "resampy~=0.4.3", + "tenacity~=9.0.0" ] [project.urls] @@ -55,7 +56,7 @@ gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.57.2" ] krisp = [ "pipecat-ai-krisp~=0.3.0" ] langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] -livekit = [ "livekit~=0.18.2", "livekit-api~=0.8.0", "tenacity~=8.5.0" ] +livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1" ] lmnt = [ "lmnt~=1.1.4" ] local = [ "pyaudio~=0.2.14" ] moondream = [ "einops~=0.8.0", "timm~=1.0.8", "transformers~=4.44.0" ] diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 8683fd29a..88fbf29d4 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -12,6 +12,8 @@ from typing import AsyncGenerator, List, Optional, Union from loguru import logger from pydantic import BaseModel +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential + from pipecat.frames.frames import ( BotStoppedSpeakingFrame, @@ -239,52 +241,64 @@ class CartesiaTTSService(WordTTSService): msg = self._build_msg(text="", continue_transcript=False) await self._websocket.send(msg) + async def _receive_messages(self): + async for message in self._get_websocket(): + msg = json.loads(message) + if not msg or msg["context_id"] != self._context_id: + continue + if msg["type"] == "done": + await self.stop_ttfb_metrics() + # Unset _context_id but not the _context_id_start_timestamp + # because we are likely still playing out audio and need the + # timestamp to set send context frames. + self._context_id = None + await self.add_word_timestamps( + [("TTSStoppedFrame", 0), ("LLMFullResponseEndFrame", 0), ("Reset", 0)] + ) + elif msg["type"] == "timestamps": + await self.add_word_timestamps( + list(zip(msg["word_timestamps"]["words"], msg["word_timestamps"]["start"])) + ) + elif msg["type"] == "chunk": + await self.stop_ttfb_metrics() + self.start_word_timestamps() + frame = TTSAudioRawFrame( + audio=base64.b64decode(msg["data"]), + sample_rate=self._settings["output_format"]["sample_rate"], + num_channels=1, + ) + await self.push_frame(frame) + elif msg["type"] == "error": + logger.error(f"{self} error: {msg}") + await self.push_frame(TTSStoppedFrame()) + await self.stop_all_metrics() + await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) + else: + logger.error(f"{self} error, unknown message type: {msg}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + async def _receive_task_handler(self): while True: try: - async for message in self._get_websocket(): - msg = json.loads(message) - if not msg or msg["context_id"] != self._context_id: - continue - if msg["type"] == "done": - await self.stop_ttfb_metrics() - # Unset _context_id but not the _context_id_start_timestamp - # because we are likely still playing out audio and need the - # timestamp to set send context frames. - self._context_id = None - await self.add_word_timestamps( - [("TTSStoppedFrame", 0), ("LLMFullResponseEndFrame", 0), ("Reset", 0)] - ) - elif msg["type"] == "timestamps": - await self.add_word_timestamps( - list( - zip( - msg["word_timestamps"]["words"], msg["word_timestamps"]["start"] - ) - ) - ) - elif msg["type"] == "chunk": - await self.stop_ttfb_metrics() - self.start_word_timestamps() - frame = TTSAudioRawFrame( - audio=base64.b64decode(msg["data"]), - sample_rate=self._settings["output_format"]["sample_rate"], - num_channels=1, - ) - await self.push_frame(frame) - elif msg["type"] == "error": - logger.error(f"{self} error: {msg}") - await self.push_frame(TTSStoppedFrame()) - await self.stop_all_metrics() - await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) - else: - logger.error(f"{self} error, unknown message type: {msg}") + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception: {e}") - await self._disconnect_websocket() - await self._connect_websocket() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index b829f4945..011c64308 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -11,11 +11,13 @@ from typing import Any, AsyncGenerator, Dict, List, Literal, Mapping, Optional, from loguru import logger from pydantic import BaseModel, model_validator +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, EndFrame, + ErrorFrame, Frame, LLMFullResponseEndFrame, StartFrame, @@ -352,28 +354,44 @@ class ElevenLabsTTSService(WordTTSService): except Exception as e: logger.error(f"{self} error closing websocket: {e}") + async def _receive_messages(self): + async for message in self._websocket: + msg = json.loads(message) + if msg.get("audio"): + await self.stop_ttfb_metrics() + self.start_word_timestamps() + + audio = base64.b64decode(msg["audio"]) + frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1) + await self.push_frame(frame) + if msg.get("alignment"): + word_times = calculate_word_times(msg["alignment"], self._cumulative_time) + await self.add_word_timestamps(word_times) + self._cumulative_time = word_times[-1][1] + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + async def _receive_task_handler(self): while True: try: - async for message in self._websocket: - msg = json.loads(message) - if msg.get("audio"): - await self.stop_ttfb_metrics() - self.start_word_timestamps() - - audio = base64.b64decode(msg["audio"]) - frame = TTSAudioRawFrame(audio, self._settings["sample_rate"], 1) - await self.push_frame(frame) - if msg.get("alignment"): - word_times = calculate_word_times(msg["alignment"], self._cumulative_time) - await self.add_word_timestamps(word_times) - self._cumulative_time = word_times[-1][1] + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception: {e}") - await self._disconnect_websocket() - await self._connect_websocket() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def _keepalive_task_handler(self): while True: diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 04223a1a1..5393e6653 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -8,6 +8,7 @@ import asyncio from typing import AsyncGenerator from loguru import logger +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from pipecat.frames.frames import ( CancelFrame, @@ -159,31 +160,47 @@ class LmntTTSService(TTSService): except Exception as e: logger.error(f"{self} error closing connection: {e}") + async def _receive_messages(self): + async for msg in self._connection: + if "error" in msg: + logger.error(f'{self} error: {msg["error"]}') + await self.push_frame(TTSStoppedFrame()) + await self.stop_all_metrics() + await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) + elif "audio" in msg: + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame( + audio=msg["audio"], + sample_rate=self._settings["output_format"]["sample_rate"], + num_channels=1, + ) + await self.push_frame(frame) + else: + logger.error(f"{self}: LMNT error, unknown message type: {msg}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_lmnt() + await self._connect_lmnt() + async def _receive_task_handler(self): while True: try: - async for msg in self._connection: - if "error" in msg: - logger.error(f'{self} error: {msg["error"]}') - await self.push_frame(TTSStoppedFrame()) - await self.stop_all_metrics() - await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) - elif "audio" in msg: - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame( - audio=msg["audio"], - sample_rate=self._settings["output_format"]["sample_rate"], - num_channels=1, - ) - await self.push_frame(frame) - else: - logger.error(f"{self}: LMNT error, unknown message type: {msg}") + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception: {e}") - await self._disconnect_lmnt() - await self._connect_lmnt() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: logger.debug(f"Generating TTS: [{text}]") diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 00c5d4d34..43e1739c9 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -15,6 +15,7 @@ import aiohttp import websockets from loguru import logger from pydantic import BaseModel +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential from pipecat.frames.frames import ( BotStoppedSpeakingFrame, @@ -234,35 +235,51 @@ class PlayHTTTSService(TTSService): await self.stop_all_metrics() self._request_id = None + async def _receive_messages(self): + async for message in self._get_websocket(): + if isinstance(message, bytes): + # Skip the WAV header message + if message.startswith(b"RIFF"): + continue + await self.stop_ttfb_metrics() + frame = TTSAudioRawFrame(message, self._settings["sample_rate"], 1) + await self.push_frame(frame) + else: + logger.debug(f"Received text message: {message}") + try: + msg = json.loads(message) + if "request_id" in msg and msg["request_id"] == self._request_id: + await self.push_frame(TTSStoppedFrame()) + self._request_id = None + elif "error" in msg: + logger.error(f"{self} error: {msg}") + await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) + except json.JSONDecodeError: + logger.error(f"Invalid JSON message: {message}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"{self} reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + async def _receive_task_handler(self): while True: try: - async for message in self._get_websocket(): - if isinstance(message, bytes): - # Skip the WAV header message - if message.startswith(b"RIFF"): - continue - await self.stop_ttfb_metrics() - frame = TTSAudioRawFrame(message, self._settings["sample_rate"], 1) - await self.push_frame(frame) - else: - logger.debug(f"Received text message: {message}") - try: - msg = json.loads(message) - if "request_id" in msg and msg["request_id"] == self._request_id: - await self.push_frame(TTSStoppedFrame()) - self._request_id = None - elif "error" in msg: - logger.error(f"{self} error: {msg}") - await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) - except json.JSONDecodeError: - logger.error(f"Invalid JSON message: {message}") + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() except asyncio.CancelledError: break except Exception as e: - logger.error(f"{self} exception in receive task: {e}") - await self._disconnect_websocket() - await self._connect_websocket() + message = f"{self} error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) From 0e31413851f93234a0cfd012245a2df733045beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 16 Dec 2024 19:20:34 -0800 Subject: [PATCH 25/90] pyproject: update numpy, pydantic, loguru --- pyproject.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ee258ff6..7789808ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,12 +21,13 @@ classifiers = [ ] dependencies = [ "aiohttp~=3.11.10", - "loguru~=0.7.2", + "loguru~=0.7.3", "Markdown~=3.7", - "numpy~=1.26.4", - "Pillow~=10.4.0", + "numpy~=2.1.3", + "numba~=0.61.0rc1", + "Pillow~=11.0.0", "protobuf~=5.29.1", - "pydantic~=2.8.2", + "pydantic~=2.10.3", "pyloudnorm~=0.1.1", "resampy~=0.4.3", "tenacity~=9.0.0" @@ -46,7 +47,6 @@ cartesia = [ "cartesia~=1.0.13", "websockets~=13.1" ] daily = [ "daily-python~=0.13.0" ] deepgram = [ "deepgram-sdk~=3.7.7" ] elevenlabs = [ "websockets~=13.1" ] -examples = [ "python-dotenv~=1.0.1", "flask~=3.0.3", "flask_cors~=4.0.1" ] fal = [ "fal-client~=0.4.1" ] gladia = [ "websockets~=13.1" ] google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.21.1" ] @@ -63,7 +63,7 @@ moondream = [ "einops~=0.8.0", "timm~=1.0.8", "transformers~=4.44.0" ] nim = [ "openai~=1.57.2" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "openai~=1.57.2", "websockets~=13.1", "python-deepcompare~=1.0.1" ] -openpipe = [ "openpipe~=4.38.0" ] +openpipe = [ "openpipe~=4.40.0" ] playht = [ "pyht~=0.1.8", "websockets~=13.1" ] riva = [ "nvidia-riva-client~=2.17.0" ] silero = [ "onnxruntime~=1.20.1" ] From 7faa4eb295713f4b1159b1f556c856b42240556c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 16 Dec 2024 19:21:08 -0800 Subject: [PATCH 26/90] update dev-requirements --- dev-requirements.txt | 14 +++++++------- pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index db2cad4fa..0ed6d9b05 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,9 @@ -build~=1.2.1 -grpcio-tools~=1.65.4 +build~=1.2.2 +grpcio-tools~=1.68.1 pip-tools~=7.4.1 -pyright~=1.1.376 -pytest~=8.3.2 -ruff~=0.6.7 -setuptools~=72.2.0 +pyright~=1.1.390 +pytest~=8.3.4 +ruff~=0.8.3 +setuptools~=75.6.0 setuptools_scm~=8.1.0 -python-dotenv~=1.0.1 \ No newline at end of file +python-dotenv~=1.0.1 diff --git a/pyproject.toml b/pyproject.toml index 7789808ed..b31bff39d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ nim = [ "openai~=1.57.2" ] noisereduce = [ "noisereduce~=3.0.3" ] openai = [ "openai~=1.57.2", "websockets~=13.1", "python-deepcompare~=1.0.1" ] openpipe = [ "openpipe~=4.40.0" ] -playht = [ "pyht~=0.1.8", "websockets~=13.1" ] +playht = [ "pyht~=0.1.9", "websockets~=13.1" ] riva = [ "nvidia-riva-client~=2.17.0" ] silero = [ "onnxruntime~=1.20.1" ] simli = [ "simli-ai~=0.1.7"] From d6bac77b3c33c989cff20b6d70b50877617e0637 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 16 Dec 2024 19:22:59 -0800 Subject: [PATCH 27/90] pyproject: add audioop-lts for python 3.13 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index b31bff39d..1a43b1b89 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ ] dependencies = [ "aiohttp~=3.11.10", + "audioop-lts~=0.2.1; python_version>='3.13'", "loguru~=0.7.3", "Markdown~=3.7", "numpy~=2.1.3", From da15c83babf9ebc1939ac8b38efff079e4ce4a7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 16 Dec 2024 20:52:40 -0800 Subject: [PATCH 28/90] fix ruff formatting --- .../metrics/frame_processor_metrics.py | 6 ++++++ src/pipecat/processors/metrics/sentry.py | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/pipecat/processors/metrics/frame_processor_metrics.py b/src/pipecat/processors/metrics/frame_processor_metrics.py index a22639239..2c0099989 100644 --- a/src/pipecat/processors/metrics/frame_processor_metrics.py +++ b/src/pipecat/processors/metrics/frame_processor_metrics.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import time from pipecat.frames.frames import MetricsFrame diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index e37dd9d44..5c53ff526 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import time from loguru import logger @@ -29,8 +35,10 @@ class SentryMetrics(FrameProcessorMetrics): description=f"TTFB for {self._processor_name()}", start_timestamp=self._start_ttfb_time, ) - logger.debug(f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: { - self._ttfb_metrics_span.description} started.") + logger.debug( + f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: { + self._ttfb_metrics_span.description} started." + ) self._should_report_ttfb = not report_only_initial_ttfb async def stop_ttfb_metrics(self): @@ -46,8 +54,10 @@ class SentryMetrics(FrameProcessorMetrics): description=f"Processing for {self._processor_name()}", start_timestamp=self._start_processing_time, ) - logger.debug(f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: { - self._processing_metrics_span.description} started.") + logger.debug( + f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: { + self._processing_metrics_span.description} started." + ) async def stop_processing_metrics(self): stop_time = time.time() From fe0a7d07bdb5a333d0605ff3acda42410c567e70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 16 Dec 2024 21:02:38 -0800 Subject: [PATCH 29/90] update CHANGELOG --- CHANGELOG.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0d424b18..53a2871aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,15 +9,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Add support for more languages to ElevenLabs (Arabic, Croatian, Filipino, +- Pipecat now supports Python 3.13. We had a dependency on the `audioop` package + which was deprecated and now removed on Python 3.13. We are now using + `audioop-lts` (https://github.com/AbstractUmbra/audioop) to provide the same + functionality. + +- Added support for more languages to ElevenLabs (Arabic, Croatian, Filipino, Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa). ### Changed -- Changed: Room expiration (`exp`) in `DailyRoomProperties` is now optional - (None) by default instead of automatically setting a 5-minute expiration - time. You must explicitly set expiration time if desired. +- Room expiration (`exp`) in `DailyRoomProperties` is now optional (`None`) by + default instead of automatically setting a 5-minute expiration time. You must + explicitly set expiration time if desired. ### Deprecated From ca086a856f40f8ec2eb122156b7ca61873d7925d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 17 Dec 2024 09:11:21 -0500 Subject: [PATCH 30/90] Add custom assistant context aggregator for Grok due to content requirement in function calling --- .../foundational/14j-function-calling-nim.py | 8 +- src/pipecat/services/grok.py | 105 +++++++++++++++++- src/pipecat/services/openai.py | 1 - 3 files changed, 107 insertions(+), 7 deletions(-) diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 68bb3b52e..4059bc386 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -65,7 +65,7 @@ async def main(): ) llm = NimLLMService( - api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.1-405b-instruct" + api_key=os.getenv("NVIDIA_API_KEY"), model="meta/llama-3.3-70b-instruct" ) # Register a function_name of None to get all functions # sent to the same callback with an additional function_name parameter. @@ -76,18 +76,18 @@ async def main(): type="function", function={ "name": "get_current_weather", - "description": "Get the current weather", + "description": "Returns the current weather at a location, if one is specified, and defaults to the user's location.", "parameters": { "type": "object", "properties": { "location": { "type": "string", - "description": "The city and state, e.g. San Francisco, CA", + "description": "The location to find the weather of, or if not provided, it's the default location.", }, "format": { "type": "string", "enum": ["celsius", "fahrenheit"], - "description": "The temperature unit to use. Infer this from the users location.", + "description": "Whether to use SI or USCS units (celsius or fahrenheit).", }, }, "required": ["location", "format"], diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index 505dfcca5..a6a1b3a64 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -5,11 +5,102 @@ # +import json +from dataclasses import dataclass + from loguru import logger from pipecat.metrics.metrics import LLMTokenUsage -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.openai import OpenAILLMService +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.services.openai import ( + OpenAIAssistantContextAggregator, + OpenAILLMService, + OpenAIUserContextAggregator, +) + + +class GrokAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Custom assistant context aggregator for Grok that handles empty content requirement.""" + + async def _push_aggregation(self): + if not ( + self._aggregation or self._function_call_result or self._pending_image_frame_message + ): + return + + run_llm = False + + aggregation = self._aggregation + self._reset() + + try: + if self._function_call_result: + frame = self._function_call_result + self._function_call_result = None + if frame.result: + # Grok requires an empty content field for function calls + self._context.add_message( + { + "role": "assistant", + "content": "", # Required by Grok + "tool_calls": [ + { + "id": frame.tool_call_id, + "function": { + "name": frame.function_name, + "arguments": json.dumps(frame.arguments), + }, + "type": "function", + } + ], + } + ) + self._context.add_message( + { + "role": "tool", + "content": json.dumps(frame.result), + "tool_call_id": frame.tool_call_id, + } + ) + # Only run the LLM if there are no more function calls in progress. + run_llm = not bool(self._function_calls_in_progress) + else: + self._context.add_message({"role": "assistant", "content": aggregation}) + + if self._pending_image_frame_message: + frame = self._pending_image_frame_message + self._pending_image_frame_message = None + self._context.add_image_frame_message( + format=frame.user_image_raw_frame.format, + size=frame.user_image_raw_frame.size, + image=frame.user_image_raw_frame.image, + text=frame.text, + ) + run_llm = True + + if run_llm: + await self._user_context_aggregator.push_context_frame() + + frame = OpenAILLMContextFrame(self._context) + await self.push_frame(frame) + + except Exception as e: + logger.error(f"Error processing frame: {e}") + + +@dataclass +class GrokContextAggregatorPair: + _user: "OpenAIUserContextAggregator" + _assistant: "GrokAssistantContextAggregator" + + def user(self) -> "OpenAIUserContextAggregator": + return self._user + + def assistant(self) -> "GrokAssistantContextAggregator": + return self._assistant class GrokLLMService(OpenAILLMService): @@ -101,3 +192,13 @@ class GrokLLMService(OpenAILLMService): # Update completion tokens count if it has increased if tokens.completion_tokens > self._completion_tokens: self._completion_tokens = tokens.completion_tokens + + @staticmethod + def create_context_aggregator( + context: OpenAILLMContext, *, assistant_expect_stripped_words: bool = True + ) -> GrokContextAggregatorPair: + user = OpenAIUserContextAggregator(context) + assistant = GrokAssistantContextAggregator( + user, expect_stripped_words=assistant_expect_stripped_words + ) + return GrokContextAggregatorPair(_user=user, _assistant=assistant) diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 43ad16536..85e1a95f0 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -559,7 +559,6 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): self._context.add_message( { "role": "assistant", - "content": "", # content field required for Grok function calling "tool_calls": [ { "id": frame.tool_call_id, From 141b0a656010c93e2213979d3b7b00d3c49d4423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 07:14:31 -0800 Subject: [PATCH 31/90] sentry: fix formatting --- src/pipecat/processors/metrics/sentry.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index 5c53ff526..6cc6d1103 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -36,8 +36,7 @@ class SentryMetrics(FrameProcessorMetrics): start_timestamp=self._start_ttfb_time, ) logger.debug( - f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: { - self._ttfb_metrics_span.description} started." + f"Sentry Span ID: {self._ttfb_metrics_span.span_id} Description: {self._ttfb_metrics_span.description} started." ) self._should_report_ttfb = not report_only_initial_ttfb @@ -55,8 +54,7 @@ class SentryMetrics(FrameProcessorMetrics): start_timestamp=self._start_processing_time, ) logger.debug( - f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: { - self._processing_metrics_span.description} started." + f"Sentry Span ID: {self._processing_metrics_span.span_id} Description: {self._processing_metrics_span.description} started." ) async def stop_processing_metrics(self): From 5f7d28bb052886cd794580fadc54fe57acf8b6c0 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 17 Dec 2024 22:07:35 +0530 Subject: [PATCH 32/90] adding type check and value check --- src/pipecat/services/deepgram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 48193388d..6404dce80 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -140,7 +140,9 @@ class DeepgramSTTService(STTService): merged_options = LiveOptions(**{**default_options.to_dict(), **live_options.to_dict()}) # deepgram connection requires language to be a string - merged_options.language = merged_options.language.value + if isinstance(merged_options.language, Language) and hasattr(merged_options.language, "value"): + merged_options.language = merged_options.language.value + self._settings = merged_options.to_dict() self._client = DeepgramClient( From b94b10f7d6aadd2e78f3674b651448cbbffe21ab Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 17 Dec 2024 22:11:52 +0530 Subject: [PATCH 33/90] added change log --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53a2871aa..78e069fc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 only be pushed downstream after the audio generated from `TTSSpeakFrame` has been spoken. +- Fixed a [bug](https://github.com/pipecat-ai/pipecat/issues/868) in `DeepgramSTTService` that was causing Language to be passed + as python object instead of a string, causing connection to fail. + ## [0.0.51] - 2024-12-16 ### Fixed From 7351e281e215d552eef324ffb941bf990e1a6331 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 17 Dec 2024 22:21:56 +0530 Subject: [PATCH 34/90] ruff change --- src/pipecat/services/deepgram.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 6404dce80..17d4ef933 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -140,7 +140,9 @@ class DeepgramSTTService(STTService): merged_options = LiveOptions(**{**default_options.to_dict(), **live_options.to_dict()}) # deepgram connection requires language to be a string - if isinstance(merged_options.language, Language) and hasattr(merged_options.language, "value"): + if isinstance(merged_options.language, Language) and hasattr( + merged_options.language, "value" + ): merged_options.language = merged_options.language.value self._settings = merged_options.to_dict() From f14d32d09e02d2edc8c50f8269cc7582ab7dc695 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 17 Dec 2024 23:11:18 +0530 Subject: [PATCH 35/90] fixing ruff issue --- src/pipecat/services/deepgram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 17d4ef933..8d3def5be 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -141,7 +141,7 @@ class DeepgramSTTService(STTService): # deepgram connection requires language to be a string if isinstance(merged_options.language, Language) and hasattr( - merged_options.language, "value" + merged_options.language, "value" ): merged_options.language = merged_options.language.value From 4208d2d7c4219927f18092889ad49f0f4328c353 Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Tue, 17 Dec 2024 23:38:36 +0530 Subject: [PATCH 36/90] updating readme to support auto-formatting of ruff in pycharm --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index f63d85167..9f2d59c86 100644 --- a/README.md +++ b/README.md @@ -224,6 +224,15 @@ Install the } ``` +### PyCharm + +`ruff` was installed in the `venv` environment described before, now to enable autoformatting on save, go to `File` -> `Settings` -> `Tools` -> `File Watchers` and add a new watcher with the following settings: +1. **Name**: `Ruff formatter` +2. **File type**: `Python` +3. **Working directory**: `$ContentRoot$` +4. **Arguments**: `format $FilePath$ --config pyproject.toml` +5. **Program**: `$PyInterpreterDirectory$/ruff` + ## Contributing We welcome contributions from the community! Whether you're fixing bugs, improving documentation, or adding new features, here's how you can help: From 53049adeeabc79a67bfe1f3c1646b5f582c7222c Mon Sep 17 00:00:00 2001 From: Vaibhav159 Date: Wed, 18 Dec 2024 00:47:00 +0530 Subject: [PATCH 37/90] removing --config flag --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f2d59c86..01e8dfea5 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,7 @@ Install the 1. **Name**: `Ruff formatter` 2. **File type**: `Python` 3. **Working directory**: `$ContentRoot$` -4. **Arguments**: `format $FilePath$ --config pyproject.toml` +4. **Arguments**: `format $FilePath$` 5. **Program**: `$PyInterpreterDirectory$/ruff` ## Contributing From 6244124d145b561289f9a52fed62d8ba683b68a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 11:19:02 -0800 Subject: [PATCH 38/90] README: added Emacs import re-organization with Ruff --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index f63d85167..b4c67cfdf 100644 --- a/README.md +++ b/README.md @@ -197,9 +197,7 @@ You can use [use-package](https://github.com/jwiegley/use-package) to install [e :hook ((python-mode . lazy-ruff-mode)) :config (setq lazy-ruff-format-command "ruff format") - (setq lazy-ruff-only-format-block t) - (setq lazy-ruff-only-format-region t) - (setq lazy-ruff-only-format-buffer t)) + (setq lazy-ruff-check-command "ruff check --select I")) ``` `ruff` was installed in the `venv` environment described before, so you should be able to use [pyvenv-auto](https://github.com/ryotaro612/pyvenv-auto) to automatically load that environment inside Emacs. @@ -209,7 +207,6 @@ You can use [use-package](https://github.com/jwiegley/use-package) to install [e :ensure t :defer t :hook ((python-mode . pyvenv-auto-run))) - ``` ### Visual Studio Code From da3fb981012479026abc18dc7f6014009563255e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 11:24:50 -0800 Subject: [PATCH 39/90] examples(storytelling-chatbot): update dependencies --- .../frontend/package-lock.json | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/storytelling-chatbot/frontend/package-lock.json b/examples/storytelling-chatbot/frontend/package-lock.json index 7b491d0f3..ce4450613 100644 --- a/examples/storytelling-chatbot/frontend/package-lock.json +++ b/examples/storytelling-chatbot/frontend/package-lock.json @@ -16,7 +16,7 @@ "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", "framer-motion": "^11.9.0", - "next": "^14.2.14", + "next": "^14.2.15", "react": "^18.3.1", "react-dom": "^18.3.1", "recoil": "^0.7.7", @@ -1912,9 +1912,9 @@ "dev": true }, "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", + "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", @@ -4047,9 +4047,9 @@ } }, "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", "funding": [ { "type": "github", From 17162258a29addf4f2f0cebcd0ee8697055b9f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 11:28:58 -0800 Subject: [PATCH 40/90] fix ruff linter import organization --- examples/chatbot-audio-recording/bot.py | 8 ++-- examples/deployment/flyio-example/bot.py | 13 +++--- .../deployment/flyio-example/bot_runner.py | 12 ++--- examples/deployment/modal-example/app.py | 3 +- examples/dialin-chatbot/bot_daily.py | 13 +++--- examples/dialin-chatbot/bot_runner.py | 12 ++--- examples/dialin-chatbot/bot_twilio.py | 14 +++--- examples/foundational/01-say-one-thing.py | 20 ++++---- examples/foundational/01a-local-audio.py | 9 ++-- examples/foundational/01b-livekit-audio.py | 9 ++-- examples/foundational/01c-fastpitch.py | 20 ++++---- examples/foundational/02-llm-say-one-thing.py | 12 ++--- examples/foundational/03-still-frame.py | 12 ++--- .../foundational/03a-local-still-frame.py | 10 ++-- .../foundational/04-utterance-and-speech.py | 15 +++--- .../foundational/05-sync-speech-and-image.py | 17 +++---- .../05a-local-sync-speech-and-image.py | 16 +++---- examples/foundational/06a-image-sync.py | 14 ++---- examples/foundational/07-interruptible-vad.py | 14 +++--- examples/foundational/07-interruptible.py | 12 ++--- .../07b-interruptible-langchain.py | 20 ++++---- .../07d-interruptible-elevenlabs.py | 2 +- .../foundational/07f-interruptible-azure.py | 13 ++---- .../07g-interruptible-openai-tts.py | 2 +- .../07h-interruptible-openpipe.py | 14 +++--- .../foundational/07i-interruptible-xtts.py | 12 ++--- .../foundational/07k-interruptible-lmnt.py | 12 ++--- .../foundational/07p-interruptible-krisp.py | 2 +- .../07s-interruptible-google-audio-in.py | 29 ++++++------ examples/foundational/08-bots-arguing.py | 19 ++++---- examples/foundational/09-mirror.py | 14 +++--- examples/foundational/09a-local-mirror.py | 13 ++---- examples/foundational/10-wake-phrase.py | 12 ++--- examples/foundational/11-sound-effects.py | 12 ++--- examples/foundational/12-describe-video.py | 12 ++--- .../12a-describe-video-gemini-flash.py | 12 ++--- .../foundational/12b-describe-video-gpt-4o.py | 12 ++--- .../12c-describe-video-anthropic.py | 14 +++--- .../foundational/13-whisper-transcription.py | 12 ++--- examples/foundational/13a-whisper-local.py | 7 ++- .../13b-deepgram-transcription.py | 14 +++--- examples/foundational/14-function-calling.py | 15 +++--- .../14a-function-calling-anthropic.py | 14 +++--- .../14b-function-calling-anthropic-video.py | 14 +++--- .../14d-function-calling-video.py | 15 +++--- .../14e-function-calling-gemini.py | 12 ++--- examples/foundational/15-switch-voices.py | 17 +++---- examples/foundational/15a-switch-languages.py | 16 +++---- examples/foundational/17-detect-user-idle.py | 12 ++--- examples/foundational/18-gstreamer-filesrc.py | 14 +++--- .../18a-gstreamer-videotestsrc.py | 12 ++--- .../20a-persistent-context-openai.py | 3 +- .../20c-persistent-context-anthropic.py | 3 +- .../20d-persistent-context-gemini.py | 1 - examples/foundational/21-tavus-layer.py | 14 +++--- .../foundational/22-natural-conversation.py | 14 +++--- .../22b-natural-conversation-proposal.py | 46 +++++++++---------- .../22d-natural-conversation-gemini-audio.py | 46 +++++++++---------- .../foundational/23-bot-background-sound.py | 14 +++--- examples/foundational/25-google-audio-in.py | 31 ++++++------- .../foundational/26-gemini-multimodal-live.py | 6 +-- examples/foundational/27-simli-layer.py | 18 ++++---- examples/foundational/runner.py | 3 +- examples/moondream-chatbot/bot.py | 15 +++--- examples/moondream-chatbot/runner.py | 3 +- examples/moondream-chatbot/server.py | 7 ++- examples/patient-intake/bot.py | 14 +++--- examples/patient-intake/runner.py | 3 +- examples/patient-intake/server.py | 7 ++- .../storytelling-chatbot/src/bot_runner.py | 18 +++----- .../storytelling-chatbot/src/processors.py | 5 +- .../storytelling-chatbot/src/utils/helpers.py | 1 + examples/studypal/runner.py | 3 +- examples/studypal/studypal.py | 17 +++---- examples/translation-chatbot/bot.py | 12 ++--- examples/translation-chatbot/server.py | 7 ++- examples/twilio-chatbot/bot.py | 21 ++++----- examples/twilio-chatbot/server.py | 4 +- examples/websocket-server/bot.py | 7 ++- src/pipecat/audio/filters/krisp_filter.py | 5 +- .../audio/filters/noisereduce_filter.py | 4 +- src/pipecat/audio/utils.py | 1 + src/pipecat/audio/vad/silero.py | 3 +- src/pipecat/metrics/metrics.py | 1 + src/pipecat/pipeline/base_pipeline.py | 1 - src/pipecat/pipeline/parallel_pipeline.py | 7 ++- src/pipecat/pipeline/runner.py | 4 +- .../pipeline/sync_parallel_pipeline.py | 5 +- src/pipecat/pipeline/task.py | 6 +-- .../pipeline/to_be_updated/merge_pipeline.py | 1 + src/pipecat/processors/aggregators/gated.py | 4 +- .../processors/aggregators/user_response.py | 2 +- src/pipecat/processors/async_generator.py | 3 +- src/pipecat/processors/audio/vad/silero.py | 4 +- .../processors/filters/wake_check_filter.py | 5 +- .../processors/frameworks/langchain.py | 4 +- .../processors/gstreamer/pipeline_source.py | 3 +- .../processors/idle_frame_processor.py | 1 - src/pipecat/processors/logger.py | 8 ++-- .../metrics/frame_processor_metrics.py | 4 +- src/pipecat/processors/metrics/sentry.py | 1 + src/pipecat/serializers/livekit.py | 4 +- src/pipecat/serializers/protobuf.py | 5 +- src/pipecat/serializers/twilio.py | 2 +- src/pipecat/services/canonical.py | 7 ++- src/pipecat/services/cartesia.py | 1 - src/pipecat/services/fal.py | 10 ++-- .../services/gemini_multimodal_live/events.py | 5 +- src/pipecat/services/moondream.py | 9 ++-- .../services/openai_realtime_beta/context.py | 2 +- .../services/openai_realtime_beta/openai.py | 8 ++-- src/pipecat/services/openpipe.py | 9 ++-- src/pipecat/services/simli.py | 14 +++--- src/pipecat/services/tavus.py | 16 +++---- .../to_be_updated/cloudflare_ai_service.py | 3 +- .../to_be_updated/google_ai_service.py | 5 +- .../services/to_be_updated/mock_ai_service.py | 3 +- src/pipecat/services/whisper.py | 4 +- src/pipecat/transports/base_output.py | 2 +- src/pipecat/transports/base_transport.py | 7 +-- src/pipecat/transports/local/audio.py | 5 +- src/pipecat/transports/local/tk.py | 6 +-- .../transports/network/fastapi_websocket.py | 5 +- .../transports/network/websocket_server.py | 5 +- src/pipecat/transports/services/livekit.py | 3 +- src/pipecat/utils/test_frame_processor.py | 1 + tests/integration/integration_azure_llm.py | 12 ++--- tests/integration/integration_ollama_llm.py | 12 ++--- tests/integration/integration_openai_llm.py | 9 ++-- tests/test_aggregators.py | 13 ++---- tests/test_ai_services.py | 3 +- tests/test_langchain.py | 6 +-- tests/test_pipeline.py | 7 ++- 133 files changed, 556 insertions(+), 722 deletions(-) diff --git a/examples/chatbot-audio-recording/bot.py b/examples/chatbot-audio-recording/bot.py index f020a9626..ee679ffcf 100644 --- a/examples/chatbot-audio-recording/bot.py +++ b/examples/chatbot-audio-recording/bot.py @@ -4,15 +4,15 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiofiles import asyncio +import datetime import io import os import sys - -import aiohttp -import datetime import wave + +import aiofiles +import aiohttp from dotenv import load_dotenv from loguru import logger from runner import configure diff --git a/examples/deployment/flyio-example/bot.py b/examples/deployment/flyio-example/bot.py index 7c69f62bd..05e55016f 100644 --- a/examples/deployment/flyio-example/bot.py +++ b/examples/deployment/flyio-example/bot.py @@ -1,22 +1,21 @@ +import argparse import asyncio import os import sys -import argparse + +from dotenv import load_dotenv +from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame, EndFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.openai import OpenAILLMService from pipecat.services.elevenlabs import ElevenLabsTTSService +from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/deployment/flyio-example/bot_runner.py b/examples/deployment/flyio-example/bot_runner.py index 3795847e4..fcdfa65ab 100644 --- a/examples/deployment/flyio-example/bot_runner.py +++ b/examples/deployment/flyio-example/bot_runner.py @@ -4,26 +4,24 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse -import subprocess import os - +import subprocess from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse from pipecat.transports.services.helpers.daily_rest import ( DailyRESTHelper, DailyRoomObject, - DailyRoomProperties, DailyRoomParams, + DailyRoomProperties, ) -from dotenv import load_dotenv - load_dotenv(override=True) diff --git a/examples/deployment/modal-example/app.py b/examples/deployment/modal-example/app.py index 97a9f32ab..21d85f79f 100644 --- a/examples/deployment/modal-example/app.py +++ b/examples/deployment/modal-example/app.py @@ -2,12 +2,11 @@ import os import aiohttp import modal +from bot import _voice_bot_process from fastapi import HTTPException from fastapi.responses import JSONResponse from loguru import logger -from bot import _voice_bot_process - MAX_SESSION_TIME = 15 * 60 # 15 minutes app = modal.App("pipecat-modal") diff --git a/examples/dialin-chatbot/bot_daily.py b/examples/dialin-chatbot/bot_daily.py index 79e8665b3..d277011ab 100644 --- a/examples/dialin-chatbot/bot_daily.py +++ b/examples/dialin-chatbot/bot_daily.py @@ -1,21 +1,20 @@ +import argparse import asyncio import os import sys -import argparse + +from dotenv import load_dotenv +from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame, EndFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport, DailyDialinSettings - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyDialinSettings, DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/dialin-chatbot/bot_runner.py b/examples/dialin-chatbot/bot_runner.py index ac2ce79bf..050ad62b9 100644 --- a/examples/dialin-chatbot/bot_runner.py +++ b/examples/dialin-chatbot/bot_runner.py @@ -7,14 +7,14 @@ provisioning a room and starting a Pipecat bot in response. Refer to README for more information. """ -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, PlainTextResponse from twilio.twiml.voice_response import VoiceResponse @@ -22,13 +22,11 @@ from twilio.twiml.voice_response import VoiceResponse from pipecat.transports.services.helpers.daily_rest import ( DailyRESTHelper, DailyRoomObject, + DailyRoomParams, DailyRoomProperties, DailyRoomSipParams, - DailyRoomParams, ) -from dotenv import load_dotenv - load_dotenv(override=True) diff --git a/examples/dialin-chatbot/bot_twilio.py b/examples/dialin-chatbot/bot_twilio.py index 135fcf4ea..86b37381a 100644 --- a/examples/dialin-chatbot/bot_twilio.py +++ b/examples/dialin-chatbot/bot_twilio.py @@ -1,24 +1,22 @@ +import argparse import asyncio import os import sys -import argparse + +from dotenv import load_dotenv +from loguru import logger +from twilio.rest import Client from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame, EndFrame from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from twilio.rest import Client - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index f0fec28d8..290a0b80c 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -5,22 +5,20 @@ # import asyncio -import aiohttp import os import sys -from pipecat.frames.frames import EndFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.task import PipelineTask -from pipecat.pipeline.runner import PipelineRunner -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport - +import aiohttp +from dotenv import load_dotenv +from loguru import logger from runner import configure -from loguru import logger - -from dotenv import load_dotenv +from pipecat.frames.frames import EndFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index e62ab1020..ca7802b1a 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -5,10 +5,13 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import EndFrame, TTSSpeakFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,10 +20,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.audio import LocalAudioTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/01b-livekit-audio.py b/examples/foundational/01b-livekit-audio.py index 9b3efc603..ed610e027 100644 --- a/examples/foundational/01b-livekit-audio.py +++ b/examples/foundational/01b-livekit-audio.py @@ -4,6 +4,9 @@ import os import sys import aiohttp +from dotenv import load_dotenv +from livekit import api +from loguru import logger from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline @@ -12,12 +15,6 @@ from pipecat.pipeline.task import PipelineTask from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.livekit import LiveKitParams, LiveKitTransport -from livekit import api - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/01c-fastpitch.py b/examples/foundational/01c-fastpitch.py index b041e6c7b..01a68abd9 100644 --- a/examples/foundational/01c-fastpitch.py +++ b/examples/foundational/01c-fastpitch.py @@ -5,22 +5,20 @@ # import asyncio -import aiohttp import os import sys -from pipecat.frames.frames import EndFrame, TTSSpeakFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.task import PipelineTask -from pipecat.pipeline.runner import PipelineRunner -from pipecat.services.riva import FastPitchTTSService -from pipecat.transports.services.daily import DailyParams, DailyTransport - +import aiohttp +from dotenv import load_dotenv +from loguru import logger from runner import configure -from loguru import logger - -from dotenv import load_dotenv +from pipecat.frames.frames import EndFrame, TTSSpeakFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineTask +from pipecat.services.riva import FastPitchTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index ba2be607c..1e1ad2827 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,12 +21,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/03-still-frame.py b/examples/foundational/03-still-frame.py index 688146cb1..e532a4b37 100644 --- a/examples/foundational/03-still-frame.py +++ b/examples/foundational/03-still-frame.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import EndFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -16,12 +20,6 @@ from pipecat.pipeline.task import PipelineTask from pipecat.services.fal import FalImageGenService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/03a-local-still-frame.py b/examples/foundational/03a-local-still-frame.py index c06834d90..821c9979c 100644 --- a/examples/foundational/03a-local-still-frame.py +++ b/examples/foundational/03a-local-still-frame.py @@ -5,12 +5,14 @@ # import asyncio -import aiohttp import os import sys - import tkinter as tk +import aiohttp +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -19,10 +21,6 @@ from pipecat.services.fal import FalImageGenService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.tk import TkLocalTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/04-utterance-and-speech.py b/examples/foundational/04-utterance-and-speech.py index 7f63757d6..02f035485 100644 --- a/examples/foundational/04-utterance-and-speech.py +++ b/examples/foundational/04-utterance-and-speech.py @@ -8,27 +8,24 @@ # This example broken on latest pipecat and needs updating. # -import aiohttp import asyncio import os import sys -from pipecat.pipeline.merge_pipeline import SequentialMergePipeline -from pipecat.pipeline.pipeline import Pipeline +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure from pipecat.frames.frames import EndPipeFrame, LLMMessagesFrame, TextFrame +from pipecat.pipeline.merge_pipeline import SequentialMergePipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.task import PipelineTask from pipecat.services.azure import AzureLLMService, AzureTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.transport_services import TransportServiceOutput from pipecat.services.transports.daily_transport import DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 64f85930b..1648b11a7 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -5,12 +5,15 @@ # import asyncio -import aiohttp import os import sys - from dataclasses import dataclass +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import ( DataFrame, Frame, @@ -22,19 +25,13 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.sync_parallel_pipeline import SyncParallelPipeline from pipecat.pipeline.task import PipelineTask -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia import CartesiaHttpTTSService -from pipecat.services.openai import OpenAILLMService from pipecat.services.fal import FalImageGenService +from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index 4a561c073..d62d853cf 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -4,20 +4,22 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - import tkinter as tk +import aiohttp +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import ( Frame, + LLMMessagesFrame, OutputAudioRawFrame, + TextFrame, TTSAudioRawFrame, URLImageRawFrame, - LLMMessagesFrame, - TextFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -26,15 +28,11 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.sentence import SentenceAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia import CartesiaHttpTTSService -from pipecat.services.openai import OpenAILLMService from pipecat.services.fal import FalImageGenService +from pipecat.services.openai import OpenAILLMService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.tk import TkLocalTransport, TkOutputTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index eda3c61df..1406e93a2 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -5,11 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger from PIL import Image +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, OutputImageRawFrame, SystemFrame, TextFrame @@ -20,14 +23,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.cartesia import CartesiaHttpTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import DailyTransport - -from pipecat.transports.services.daily import DailyParams -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index 59013bd3a..53adf84f5 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -5,26 +5,24 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.audio.vad.silero import SileroVAD from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.audio.vad.silero import SileroVAD from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index 3148986a8..c68d4aa40 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,12 +23,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index e86c4a8e9..8bf4ceea2 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -9,6 +9,14 @@ import os import sys import aiohttp +from dotenv import load_dotenv +from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder +from langchain_community.chat_message_histories import ChatMessageHistory +from langchain_core.chat_history import BaseChatMessageHistory +from langchain_core.runnables.history import RunnableWithMessageHistory +from langchain_openai import ChatOpenAI +from loguru import logger +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -23,18 +31,6 @@ from pipecat.processors.frameworks.langchain import LangchainProcessor from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from langchain.prompts import ChatPromptTemplate, MessagesPlaceholder -from langchain_community.chat_message_histories import ChatMessageHistory -from langchain_core.chat_history import BaseChatMessageHistory -from langchain_core.runnables.history import RunnableWithMessageHistory -from langchain_openai import ChatOpenAI - -from loguru import logger - -from runner import configure - -from dotenv import load_dotenv - load_dotenv(override=True) diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index 24ccf6a24..8f3895d00 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -11,7 +11,6 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,6 +18,7 @@ from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index fb3e8ea0c..b08deb1b3 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -4,11 +4,15 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -18,13 +22,6 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.azure import AzureLLMService, AzureSTTService, AzureTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07g-interruptible-openai-tts.py b/examples/foundational/07g-interruptible-openai-tts.py index af414e3e3..17dbb2b30 100644 --- a/examples/foundational/07g-interruptible-openai-tts.py +++ b/examples/foundational/07g-interruptible-openai-tts.py @@ -11,7 +11,6 @@ import sys import aiohttp from dotenv import load_dotenv from loguru import logger -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer @@ -19,6 +18,7 @@ from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService, OpenAITTSService from pipecat.transports.services.daily import DailyParams, DailyTransport diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index f01d535df..aa4373d4b 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -5,9 +5,14 @@ # import asyncio -import aiohttp import os import sys +import time + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -19,13 +24,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openpipe import OpenPipeLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger -import time - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index 5bfe13b5c..ea20b0238 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,12 +23,6 @@ from pipecat.services.openai import OpenAILLMService from pipecat.services.xtts import XTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index 49179e747..31b65d6a5 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -4,11 +4,15 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -19,12 +23,6 @@ from pipecat.services.lmnt import LmntTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 834e59037..27070df0b 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -13,6 +13,7 @@ from dotenv import load_dotenv from loguru import logger from runner import configure +from pipecat.audio.filters.krisp_filter import KrispFilter from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -25,7 +26,6 @@ from pipecat.services.deepgram import DeepgramSTTService, DeepgramTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer -from pipecat.audio.filters.krisp_filter import KrispFilter load_dotenv(override=True) diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 1778e0c62..854cf1954 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -4,38 +4,37 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - -import google.ai.generativelanguage as glm - from dataclasses import dataclass + +import aiohttp +import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.google import GoogleLLMService -from pipecat.processors.frame_processor import FrameProcessor -from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.frames.frames import ( - LLMFullResponseStartFrame, - LLMFullResponseEndFrame, - InputAudioRawFrame, Frame, + InputAudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, StartInterruptionFrame, TextFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.google import GoogleLLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/08-bots-arguing.py b/examples/foundational/08-bots-arguing.py index 150fbfc0a..e455b321b 100644 --- a/examples/foundational/08-bots-arguing.py +++ b/examples/foundational/08-bots-arguing.py @@ -1,20 +1,19 @@ -from typing import Tuple -import aiohttp import asyncio import logging import os -from pipecat.processors.aggregators import SentenceAggregator -from pipecat.pipeline.pipeline import Pipeline +from typing import Tuple -from pipecat.transports.services.daily import DailyTransport +import aiohttp +from dotenv import load_dotenv +from runner import configure + +from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators import SentenceAggregator from pipecat.services.azure import AzureLLMService, AzureTTSService from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.fal import FalImageGenService -from pipecat.frames.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesFrame, TextFrame - -from runner import configure - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index a719d54f6..676935da6 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -4,10 +4,14 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -19,13 +23,7 @@ from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.transports.services.daily import DailyTransport, DailyParams - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index 539cca600..50b729883 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -4,12 +4,15 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import sys - import tkinter as tk +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -25,12 +28,6 @@ from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.tk import TkLocalTransport from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index 982303aee..b69ef683b 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -19,12 +23,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index d8692a7f1..a52b214e7 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -4,12 +4,16 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys import wave +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( Frame, @@ -29,12 +33,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index b5bb577aa..9003ca6f0 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.moondream import MoondreamService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index bc76afc73..df37c24a9 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.google import GoogleLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index d8474b568..4e96f7264 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index bc6f5a4ea..1c88b5dd4 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline @@ -17,16 +21,10 @@ from pipecat.pipeline.task import PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/13-whisper-transcription.py b/examples/foundational/13-whisper-transcription.py index c895cb944..72fee4bb0 100644 --- a/examples/foundational/13-whisper-transcription.py +++ b/examples/foundational/13-whisper-transcription.py @@ -4,10 +4,14 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -16,12 +20,6 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.services.whisper import WhisperSTTService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/13a-whisper-local.py b/examples/foundational/13a-whisper-local.py index c1ba37ca9..dcb5fd51a 100644 --- a/examples/foundational/13a-whisper-local.py +++ b/examples/foundational/13a-whisper-local.py @@ -7,6 +7,9 @@ import asyncio import sys +from dotenv import load_dotenv +from loguru import logger + from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -16,10 +19,6 @@ from pipecat.services.whisper import WhisperSTTService from pipecat.transports.base_transport import TransportParams from pipecat.transports.local.audio import LocalAudioTransport -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/13b-deepgram-transcription.py b/examples/foundational/13b-deepgram-transcription.py index 7b3a25316..ad464e87a 100644 --- a/examples/foundational/13b-deepgram-transcription.py +++ b/examples/foundational/13b-deepgram-transcription.py @@ -4,25 +4,23 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import Frame, TranscriptionFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.services.deepgram import DeepgramSTTService, LiveOptions, Language +from pipecat.services.deepgram import DeepgramSTTService, Language, LiveOptions from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 2479e3f3e..0aebc915a 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -5,10 +5,15 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,14 +22,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 9d3335809..24587c935 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -5,25 +5,23 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index cdc69556d..7473af10d 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -5,25 +5,23 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index e238d91eb..93a87f420 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -5,10 +5,15 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -17,14 +22,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index ede7222bb..35b3fa264 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -18,12 +22,6 @@ from pipecat.services.google import GoogleLLMService from pipecat.services.openai import OpenAILLMContext from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index 5c61dd25f..aa516fdb2 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -4,15 +4,20 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame -from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -21,14 +26,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index 4b8794ab8..86798ba50 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -5,16 +5,20 @@ # import asyncio -import aiohttp import os import sys +import aiohttp from deepgram import LiveOptions +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame -from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext @@ -24,14 +28,6 @@ from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from openai.types.chat import ChatCompletionToolParam - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index 7268e2305..a3e11fb0f 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -5,10 +5,14 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -20,12 +24,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/18-gstreamer-filesrc.py b/examples/foundational/18-gstreamer-filesrc.py index 8ebcaa1b9..169e46bc4 100644 --- a/examples/foundational/18-gstreamer-filesrc.py +++ b/examples/foundational/18-gstreamer-filesrc.py @@ -4,23 +4,21 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import asyncio -import aiohttp import argparse +import asyncio import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure_with_args + from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure_with_args - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/18a-gstreamer-videotestsrc.py b/examples/foundational/18a-gstreamer-videotestsrc.py index 9e5977348..778fea06d 100644 --- a/examples/foundational/18a-gstreamer-videotestsrc.py +++ b/examples/foundational/18a-gstreamer-videotestsrc.py @@ -5,21 +5,19 @@ # import asyncio -import aiohttp import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineTask from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index d4d418e91..bf8af53ea 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -24,9 +24,8 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) -from pipecat.services.openai import OpenAILLMService from pipecat.services.cartesia import CartesiaTTSService - +from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index 421b00603..f8a42fda3 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -24,9 +24,8 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) -from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.anthropic import AnthropicLLMService - +from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 25d809021..52736606f 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -26,7 +26,6 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.google import GoogleLLMService - from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/21-tavus-layer.py b/examples/foundational/21-tavus-layer.py index 61705b28a..abb2f202d 100644 --- a/examples/foundational/21-tavus-layer.py +++ b/examples/foundational/21-tavus-layer.py @@ -5,12 +5,15 @@ # import asyncio -import aiohttp import os import sys - from typing import Any, Mapping +import aiohttp +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -20,15 +23,10 @@ from pipecat.processors.aggregators.llm_response import ( LLMUserResponseAggregator, ) from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.openai import OpenAILLMService from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService from pipecat.services.tavus import TavusVideoService from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.audio.vad.silero import SileroVADAnalyzer - -from loguru import logger - -from dotenv import load_dotenv load_dotenv(override=True) diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 73dfb003d..6f9dbbb5b 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -5,14 +5,18 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.gated_openai_llm_context import GatedOpenAILLMContextAggregator @@ -26,12 +30,6 @@ from pipecat.services.openai import OpenAILLMService from pipecat.sync.event_notifier import EventNotifier from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 2deeb3da4..1ed4360e3 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -4,50 +4,48 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys import time +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.services.openai import OpenAILLMService -from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.processors.frame_processor import FrameProcessor, FrameDirection from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, + LLMMessagesFrame, StartFrame, StartInterruptionFrame, StopInterruptionFrame, SystemFrame, + TextFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame -from pipecat.sync.base_notifier import BaseNotifier +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.processors.filters.function_filter import FunctionFilter +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.user_idle_processor import UserIdleProcessor - - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.sync.base_notifier import BaseNotifier +from pipecat.sync.event_notifier import EventNotifier +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 1ff8aa23e..96e9e12c5 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -4,51 +4,49 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys import time +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, TextFrame -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, -) -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.google import GoogleLLMService, GoogleLLMContext -from pipecat.sync.event_notifier import EventNotifier -from pipecat.transports.services.daily import DailyParams, DailyTransport -from pipecat.processors.frame_processor import FrameProcessor, FrameDirection from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, InputAudioRawFrame, + LLMMessagesFrame, StartFrame, StartInterruptionFrame, StopInterruptionFrame, SystemFrame, + TextFrame, TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame -from pipecat.sync.base_notifier import BaseNotifier +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.processors.filters.function_filter import FunctionFilter +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.user_idle_processor import UserIdleProcessor - - -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.google import GoogleLLMContext, GoogleLLMService +from pipecat.sync.base_notifier import BaseNotifier +from pipecat.sync.event_notifier import EventNotifier +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index 68dd700d5..cde90f933 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -6,13 +6,17 @@ import argparse import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure_with_args + from pipecat.audio.mixers.soundfile_mixer import SoundfileMixer from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, MixerUpdateSettingsFrame, MixerEnableFrame +from pipecat.frames.frames import LLMMessagesFrame, MixerEnableFrame, MixerUpdateSettingsFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -21,12 +25,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure_with_args - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index 843d24e1f..477b72ddc 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -4,31 +4,18 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - -import google.ai.generativelanguage as glm - from dataclasses import dataclass + +import aiohttp +import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.pipeline.pipeline import Pipeline -from pipecat.pipeline.parallel_pipeline import ParallelPipeline -from pipecat.pipeline.runner import PipelineRunner -from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.google import GoogleLLMService, GoogleLLMContext -from pipecat.processors.frame_processor import FrameProcessor -from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -40,6 +27,18 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.pipeline.parallel_pipeline import ParallelPipeline +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.google import GoogleLLMContext, GoogleLLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 3a73c648a..3b528ec26 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -4,23 +4,21 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys - +import aiohttp from dotenv import load_dotenv from loguru import logger from runner import configure -from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService - from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.gemini_multimodal_live.gemini import GeminiMultimodalLiveLLMService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 55e6de2b7..415384380 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -5,27 +5,25 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure +from simli import SimliConfig + from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.frames.frames import LLMMessagesFrame - from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService -from pipecat.transports.services.daily import DailyParams, DailyTransport - -from runner import configure -from loguru import logger -from dotenv import load_dotenv - -from simli import SimliConfig from pipecat.services.simli import SimliVideoService +from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) diff --git a/examples/foundational/runner.py b/examples/foundational/runner.py index 13c4ff076..f4c774757 100644 --- a/examples/foundational/runner.py +++ b/examples/foundational/runner.py @@ -4,10 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index 54c2013b4..a815148b1 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -5,21 +5,24 @@ # import asyncio -import aiohttp import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger from PIL import Image +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import ( BotStartedSpeakingFrame, BotStoppedSpeakingFrame, + Frame, ImageRawFrame, + LLMMessagesFrame, OutputImageRawFrame, SpriteFrame, - Frame, - LLMMessagesFrame, TextFrame, UserImageRawFrame, UserImageRequestFrame, @@ -37,12 +40,6 @@ from pipecat.services.moondream import MoondreamService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/moondream-chatbot/runner.py b/examples/moondream-chatbot/runner.py index 3df3ee81f..f19fcf211 100644 --- a/examples/moondream-chatbot/runner.py +++ b/examples/moondream-chatbot/runner.py @@ -4,10 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/moondream-chatbot/server.py b/examples/moondream-chatbot/server.py index aa4e82c90..4caa47f5b 100644 --- a/examples/moondream-chatbot/server.py +++ b/examples/moondream-chatbot/server.py @@ -4,14 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index c33a2495d..7c65393c7 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -5,28 +5,26 @@ # import asyncio -import aiohttp import os import sys import wave +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import OutputAudioRawFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask -from pipecat.processors.logger import FrameLogger from pipecat.processors.frame_processor import FrameDirection +from pipecat.processors.logger import FrameLogger from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/patient-intake/runner.py b/examples/patient-intake/runner.py index 3df3ee81f..f19fcf211 100644 --- a/examples/patient-intake/runner.py +++ b/examples/patient-intake/runner.py @@ -4,10 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/patient-intake/server.py b/examples/patient-intake/server.py index 20894b019..2d2fee0ed 100644 --- a/examples/patient-intake/server.py +++ b/examples/patient-intake/server.py @@ -4,14 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/src/bot_runner.py index 25a1bca37..79de6c3e3 100644 --- a/examples/storytelling-chatbot/src/bot_runner.py +++ b/examples/storytelling-chatbot/src/bot_runner.py @@ -4,31 +4,27 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse -import subprocess import os - +import subprocess +from contextlib import asynccontextmanager from pathlib import Path from typing import Optional -from contextlib import asynccontextmanager - -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from dotenv import load_dotenv +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware -from fastapi.staticfiles import StaticFiles from fastapi.responses import FileResponse, JSONResponse +from fastapi.staticfiles import StaticFiles from pipecat.transports.services.helpers.daily_rest import ( DailyRESTHelper, DailyRoomObject, - DailyRoomProperties, DailyRoomParams, + DailyRoomProperties, ) - -from dotenv import load_dotenv - load_dotenv(override=True) # ------------ Fast API Config ------------ # diff --git a/examples/storytelling-chatbot/src/processors.py b/examples/storytelling-chatbot/src/processors.py index 6aa9ad7ab..096efd577 100644 --- a/examples/storytelling-chatbot/src/processors.py +++ b/examples/storytelling-chatbot/src/processors.py @@ -1,6 +1,8 @@ import re from async_timeout import timeout +from prompts import CUE_ASSISTANT_TURN, CUE_USER_TURN, IMAGE_GEN_PROMPT +from utils.helpers import load_sounds from pipecat.frames.frames import ( Frame, @@ -11,9 +13,6 @@ from pipecat.frames.frames import ( from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.services.daily import DailyTransportMessageFrame -from utils.helpers import load_sounds -from prompts import IMAGE_GEN_PROMPT, CUE_USER_TURN, CUE_ASSISTANT_TURN - sounds = load_sounds(["talking.wav", "listening.wav", "ding.wav"]) # -------------- Frame Types ------------- # diff --git a/examples/storytelling-chatbot/src/utils/helpers.py b/examples/storytelling-chatbot/src/utils/helpers.py index 36ba3e609..ac3a38ede 100644 --- a/examples/storytelling-chatbot/src/utils/helpers.py +++ b/examples/storytelling-chatbot/src/utils/helpers.py @@ -1,5 +1,6 @@ import os import wave + from PIL import Image from pipecat.frames.frames import OutputAudioRawFrame, OutputImageRawFrame diff --git a/examples/studypal/runner.py b/examples/studypal/runner.py index 13c4ff076..f4c774757 100644 --- a/examples/studypal/runner.py +++ b/examples/studypal/runner.py @@ -4,10 +4,11 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import argparse import os +import aiohttp + from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper diff --git a/examples/studypal/studypal.py b/examples/studypal/studypal.py index bccd7ad27..0bed8dedb 100644 --- a/examples/studypal/studypal.py +++ b/examples/studypal/studypal.py @@ -1,12 +1,15 @@ -import aiohttp import asyncio +import io import os import sys -import io -from bs4 import BeautifulSoup -from pypdf import PdfReader +import aiohttp import tiktoken +from bs4 import BeautifulSoup +from dotenv import load_dotenv +from loguru import logger +from pypdf import PdfReader +from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame @@ -18,12 +21,6 @@ from pipecat.services.cartesia import CartesiaTTSService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) # Run this script directly from your command line. diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index e654c0159..946864426 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -4,11 +4,15 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import asyncio import os import sys +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + from pipecat.frames.frames import Frame, LLMMessagesFrame, TextFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner @@ -25,12 +29,6 @@ from pipecat.transports.services.daily import ( DailyTransportMessageFrame, ) -from runner import configure - -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/examples/translation-chatbot/server.py b/examples/translation-chatbot/server.py index 9063e28b1..ffd8fc5d6 100644 --- a/examples/translation-chatbot/server.py +++ b/examples/translation-chatbot/server.py @@ -4,14 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp -import os import argparse +import os import subprocess - from contextlib import asynccontextmanager -from fastapi import FastAPI, Request, HTTPException +import aiohttp +from fastapi import FastAPI, HTTPException, Request from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse diff --git a/examples/twilio-chatbot/bot.py b/examples/twilio-chatbot/bot.py index 5e6d91910..57d542e24 100644 --- a/examples/twilio-chatbot/bot.py +++ b/examples/twilio-chatbot/bot.py @@ -1,24 +1,23 @@ import os import sys +from dotenv import load_dotenv +from loguru import logger + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import EndFrame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.cartesia import CartesiaTTSService -from pipecat.services.openai import OpenAILLMService -from pipecat.services.deepgram import DeepgramSTTService -from pipecat.transports.network.fastapi_websocket import ( - FastAPIWebsocketTransport, - FastAPIWebsocketParams, -) from pipecat.serializers.twilio import TwilioFrameSerializer - -from loguru import logger - -from dotenv import load_dotenv +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.network.fastapi_websocket import ( + FastAPIWebsocketParams, + FastAPIWebsocketTransport, +) load_dotenv(override=True) diff --git a/examples/twilio-chatbot/server.py b/examples/twilio-chatbot/server.py index 31e98e25f..c59b6e2ee 100644 --- a/examples/twilio-chatbot/server.py +++ b/examples/twilio-chatbot/server.py @@ -1,13 +1,11 @@ import json import uvicorn - +from bot import run_bot from fastapi import FastAPI, WebSocket from fastapi.middleware.cors import CORSMiddleware from starlette.responses import HTMLResponse -from bot import run_bot - app = FastAPI() app.add_middleware( diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index 3f961de8d..80633ac74 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -8,6 +8,9 @@ import asyncio import os import sys +from dotenv import load_dotenv +from loguru import logger + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline @@ -22,10 +25,6 @@ from pipecat.transports.network.websocket_server import ( WebsocketServerTransport, ) -from loguru import logger - -from dotenv import load_dotenv - load_dotenv(override=True) logger.remove(0) diff --git a/src/pipecat/audio/filters/krisp_filter.py b/src/pipecat/audio/filters/krisp_filter.py index 0055c672b..7a54c5bd5 100644 --- a/src/pipecat/audio/filters/krisp_filter.py +++ b/src/pipecat/audio/filters/krisp_filter.py @@ -4,11 +4,12 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import numpy as np import os -from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +import numpy as np from loguru import logger + +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame try: diff --git a/src/pipecat/audio/filters/noisereduce_filter.py b/src/pipecat/audio/filters/noisereduce_filter.py index 4f0449452..ed68bb27a 100644 --- a/src/pipecat/audio/filters/noisereduce_filter.py +++ b/src/pipecat/audio/filters/noisereduce_filter.py @@ -5,11 +5,9 @@ # import numpy as np - -from pipecat.audio.filters.base_audio_filter import BaseAudioFilter - from loguru import logger +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame try: diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py index 057942e04..5ef48dd1e 100644 --- a/src/pipecat/audio/utils.py +++ b/src/pipecat/audio/utils.py @@ -5,6 +5,7 @@ # import audioop + import numpy as np import pyloudnorm as pyln import resampy diff --git a/src/pipecat/audio/vad/silero.py b/src/pipecat/audio/vad/silero.py index 1da0fb12d..28e0de716 100644 --- a/src/pipecat/audio/vad/silero.py +++ b/src/pipecat/audio/vad/silero.py @@ -7,11 +7,10 @@ import time import numpy as np +from loguru import logger from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams -from loguru import logger - # How often should we reset internal model state _MODEL_RESET_STATES_TIME = 5.0 diff --git a/src/pipecat/metrics/metrics.py b/src/pipecat/metrics/metrics.py index 053708998..c40f68590 100644 --- a/src/pipecat/metrics/metrics.py +++ b/src/pipecat/metrics/metrics.py @@ -1,4 +1,5 @@ from typing import Optional + from pydantic import BaseModel diff --git a/src/pipecat/pipeline/base_pipeline.py b/src/pipecat/pipeline/base_pipeline.py index 393914684..a5ad68aa4 100644 --- a/src/pipecat/pipeline/base_pipeline.py +++ b/src/pipecat/pipeline/base_pipeline.py @@ -5,7 +5,6 @@ # from abc import abstractmethod - from typing import List from pipecat.processors.frame_processor import FrameProcessor diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 40bfea90d..323f7ed24 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -5,16 +5,15 @@ # import asyncio - from itertools import chain from typing import Awaitable, Callable, List +from loguru import logger + +from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame - -from loguru import logger class Source(FrameProcessor): diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index 57b818487..e83eab0f7 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -7,11 +7,11 @@ import asyncio import signal +from loguru import logger + from pipecat.pipeline.task import PipelineTask from pipecat.utils.utils import obj_count, obj_id -from loguru import logger - class PipelineRunner: def __init__(self, *, name: str | None = None, handle_sigint: bool = True): diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py index 20f4275e4..5f9ff9ce7 100644 --- a/src/pipecat/pipeline/sync_parallel_pipeline.py +++ b/src/pipecat/pipeline/sync_parallel_pipeline.py @@ -5,18 +5,17 @@ # import asyncio - from dataclasses import dataclass from itertools import chain from typing import List +from loguru import logger + from pipecat.frames.frames import ControlFrame, EndFrame, Frame, SystemFrame from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - @dataclass class SyncFrame(ControlFrame): diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index f09013a58..2ed5afcee 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -5,9 +5,9 @@ # import asyncio - from typing import AsyncIterable, Iterable +from loguru import logger from pydantic import BaseModel from pipecat.clocks.base_clock import BaseClock @@ -23,13 +23,11 @@ from pipecat.frames.frames import ( StartFrame, StopTaskFrame, ) -from pipecat.metrics.metrics import TTFBMetricsData, ProcessingMetricsData +from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.utils.utils import obj_count, obj_id -from loguru import logger - class PipelineParams(BaseModel): allow_interruptions: bool = False diff --git a/src/pipecat/pipeline/to_be_updated/merge_pipeline.py b/src/pipecat/pipeline/to_be_updated/merge_pipeline.py index 6142a55ea..27a52894b 100644 --- a/src/pipecat/pipeline/to_be_updated/merge_pipeline.py +++ b/src/pipecat/pipeline/to_be_updated/merge_pipeline.py @@ -1,4 +1,5 @@ from typing import List + from pipecat.frames.frames import EndFrame, EndPipeFrame from pipecat.pipeline.pipeline import Pipeline diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index c39a35c82..6976c9f10 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -6,11 +6,11 @@ from typing import List, Tuple +from loguru import logger + from pipecat.frames.frames import Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - class GatedAggregator(FrameProcessor): """Accumulate frames, with custom functions to start and stop accumulation. diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index 903019059..fd6b607a5 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.frames.frames import ( Frame, InterimTranscriptionFrame, @@ -14,6 +13,7 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, ) +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class ResponseAggregator(FrameProcessor): diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py index 4f9bc85d0..892f9d8b8 100644 --- a/src/pipecat/processors/async_generator.py +++ b/src/pipecat/processors/async_generator.py @@ -5,7 +5,6 @@ # import asyncio - from typing import Any, AsyncGenerator from pipecat.frames.frames import ( @@ -13,7 +12,7 @@ from pipecat.frames.frames import ( EndFrame, Frame, ) -from pipecat.processors.frame_processor import FrameProcessor, FrameDirection +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.serializers.base_serializer import FrameSerializer diff --git a/src/pipecat/processors/audio/vad/silero.py b/src/pipecat/processors/audio/vad/silero.py index 4aa32a163..2b115a8bb 100644 --- a/src/pipecat/processors/audio/vad/silero.py +++ b/src/pipecat/processors/audio/vad/silero.py @@ -4,6 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # +from loguru import logger + from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.vad_analyzer import VADParams, VADState from pipecat.frames.frames import ( @@ -16,8 +18,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - class SileroVAD(FrameProcessor): def __init__( diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index f1a7afbef..860e45fa3 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -6,14 +6,13 @@ import re import time - from enum import Enum +from loguru import logger + from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - class WakeCheckFilter(FrameProcessor): """ diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index c0b657244..47789cec9 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -6,6 +6,8 @@ from typing import Union +from loguru import logger + from pipecat.frames.frames import ( Frame, LLMFullResponseEndFrame, @@ -15,8 +17,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - try: from langchain_core.messages import AIMessageChunk from langchain_core.runnables import Runnable diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 649a2c529..7d2c23c69 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -6,6 +6,7 @@ import asyncio +from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( @@ -19,8 +20,6 @@ from pipecat.frames.frames import ( ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger - try: import gi diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index e674b6b84..d1c86e3ab 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -5,7 +5,6 @@ # import asyncio - from typing import Awaitable, Callable, List from pipecat.frames.frames import Frame diff --git a/src/pipecat/processors/logger.py b/src/pipecat/processors/logger.py index a26c67014..8c925af2a 100644 --- a/src/pipecat/processors/logger.py +++ b/src/pipecat/processors/logger.py @@ -4,11 +4,13 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.frames.frames import BotSpeakingFrame, Frame, AudioRawFrame, TransportMessageFrame -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from loguru import logger from typing import Optional +from loguru import logger + +from pipecat.frames.frames import AudioRawFrame, BotSpeakingFrame, Frame, TransportMessageFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + logger = logger.opt(ansi=True) diff --git a/src/pipecat/processors/metrics/frame_processor_metrics.py b/src/pipecat/processors/metrics/frame_processor_metrics.py index 2c0099989..9cacb9a7f 100644 --- a/src/pipecat/processors/metrics/frame_processor_metrics.py +++ b/src/pipecat/processors/metrics/frame_processor_metrics.py @@ -6,6 +6,8 @@ import time +from loguru import logger + from pipecat.frames.frames import MetricsFrame from pipecat.metrics.metrics import ( LLMTokenUsage, @@ -16,8 +18,6 @@ from pipecat.metrics.metrics import ( TTSUsageMetricsData, ) -from loguru import logger - class FrameProcessorMetrics: def __init__(self): diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index 6cc6d1103..bc93170ff 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -5,6 +5,7 @@ # import time + from loguru import logger try: diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py index a14483b15..aafc00b23 100644 --- a/src/pipecat/serializers/livekit.py +++ b/src/pipecat/serializers/livekit.py @@ -7,11 +7,11 @@ import ctypes import pickle +from loguru import logger + from pipecat.frames.frames import Frame, InputAudioRawFrame, OutputAudioRawFrame from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType -from loguru import logger - try: from livekit.rtc import AudioFrame except ModuleNotFoundError as e: diff --git a/src/pipecat/serializers/protobuf.py b/src/pipecat/serializers/protobuf.py index 1d2e6c3f2..4e6ade772 100644 --- a/src/pipecat/serializers/protobuf.py +++ b/src/pipecat/serializers/protobuf.py @@ -6,8 +6,9 @@ import dataclasses -import pipecat.frames.protobufs.frames_pb2 as frame_protos +from loguru import logger +import pipecat.frames.protobufs.frames_pb2 as frame_protos from pipecat.frames.frames import ( Frame, InputAudioRawFrame, @@ -17,8 +18,6 @@ from pipecat.frames.frames import ( ) from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType -from loguru import logger - class ProtobufFrameSerializer(FrameSerializer): SERIALIZABLE_TYPES = { diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py index a0d02fa2f..3c3648365 100644 --- a/src/pipecat/serializers/twilio.py +++ b/src/pipecat/serializers/twilio.py @@ -9,7 +9,7 @@ import json from pydantic import BaseModel -from pipecat.audio.utils import ulaw_to_pcm, pcm_to_ulaw +from pipecat.audio.utils import pcm_to_ulaw, ulaw_to_pcm from pipecat.frames.frames import AudioRawFrame, Frame, InputAudioRawFrame, StartInterruptionFrame from pipecat.serializers.base_serializer import FrameSerializer, FrameSerializerType diff --git a/src/pipecat/services/canonical.py b/src/pipecat/services/canonical.py index 265cc1b1b..376168c3f 100644 --- a/src/pipecat/services/canonical.py +++ b/src/pipecat/services/canonical.py @@ -4,23 +4,22 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import io import os import uuid import wave - from datetime import datetime from typing import Dict, List, Tuple +import aiohttp +from loguru import logger + from pipecat.frames.frames import CancelFrame, EndFrame, Frame from pipecat.processors.audio import audio_buffer_processor from pipecat.processors.audio.audio_buffer_processor import AudioBufferProcessor from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AIService -from loguru import logger - try: import aiofiles import aiofiles.os diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index 88fbf29d4..f4d5935a8 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -14,7 +14,6 @@ from loguru import logger from pydantic import BaseModel from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential - from pipecat.frames.frames import ( BotStoppedSpeakingFrame, CancelFrame, diff --git a/src/pipecat/services/fal.py b/src/pipecat/services/fal.py index aecdeb709..a7b2b7e30 100644 --- a/src/pipecat/services/fal.py +++ b/src/pipecat/services/fal.py @@ -4,20 +4,18 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import aiohttp import io import os +from typing import AsyncGenerator, Dict, Optional, Union +import aiohttp +from loguru import logger +from PIL import Image from pydantic import BaseModel -from typing import AsyncGenerator, Optional, Union, Dict from pipecat.frames.frames import ErrorFrame, Frame, URLImageRawFrame from pipecat.services.ai_services import ImageGenService -from PIL import Image - -from loguru import logger - try: import fal_client except ModuleNotFoundError as e: diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index 24e2b015e..edb2867f9 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -6,13 +6,12 @@ # import base64 -import json import io - -from pydantic import BaseModel, Field +import json from typing import List, Literal, Optional from PIL import Image +from pydantic import BaseModel, Field from pipecat.frames.frames import ImageRawFrame diff --git a/src/pipecat/services/moondream.py b/src/pipecat/services/moondream.py index 74442dfee..400028700 100644 --- a/src/pipecat/services/moondream.py +++ b/src/pipecat/services/moondream.py @@ -5,19 +5,16 @@ # import asyncio - -from PIL import Image - from typing import AsyncGenerator +from loguru import logger +from PIL import Image + from pipecat.frames.frames import ErrorFrame, Frame, TextFrame, VisionImageRawFrame from pipecat.services.ai_services import VisionService -from loguru import logger - try: import torch - from transformers import AutoModelForCausalLM, AutoTokenizer except ModuleNotFoundError as e: logger.error(f"Exception: {e}") diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 2b6ff968f..9ea8dd691 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -21,7 +21,7 @@ from pipecat.services.openai import ( ) from . import events -from .frames import RealtimeMessagesUpdateFrame, RealtimeFunctionCallResultFrame +from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame class OpenAIRealtimeLLMContext(OpenAILLMContext): diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index ac492a205..c0240676e 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -8,10 +8,10 @@ import asyncio import base64 import json import time - from dataclasses import dataclass import websockets +from loguru import logger from pipecat.frames.frames import ( BotStoppedSpeakingFrame, @@ -48,13 +48,11 @@ from pipecat.utils.time import time_now_iso8601 from . import events from .context import ( + OpenAIRealtimeAssistantContextAggregator, OpenAIRealtimeLLMContext, OpenAIRealtimeUserContextAggregator, - OpenAIRealtimeAssistantContextAggregator, ) -from .frames import RealtimeMessagesUpdateFrame, RealtimeFunctionCallResultFrame - -from loguru import logger +from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame @dataclass diff --git a/src/pipecat/services/openpipe.py b/src/pipecat/services/openpipe.py index 827ffcb85..fb6362b15 100644 --- a/src/pipecat/services/openpipe.py +++ b/src/pipecat/services/openpipe.py @@ -6,14 +6,15 @@ from typing import Dict, List +from loguru import logger + from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService -from loguru import logger - try: - from openpipe import AsyncOpenAI as OpenPipeAI, AsyncStream - from openai.types.chat import ChatCompletionMessageParam, ChatCompletionChunk + from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam + from openpipe import AsyncOpenAI as OpenPipeAI + from openpipe import AsyncStream except ModuleNotFoundError as e: logger.error(f"Exception: {e}") logger.error( diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index bfae861dc..1f88838be 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -6,24 +6,22 @@ import asyncio +import numpy as np +from loguru import logger + from pipecat.frames.frames import ( + CancelFrame, + EndFrame, Frame, OutputImageRawFrame, - TTSAudioRawFrame, StartInterruptionFrame, - EndFrame, - CancelFrame, + TTSAudioRawFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor, StartFrame -import numpy as np - -from loguru import logger - try: from av.audio.frame import AudioFrame from av.audio.resampler import AudioResampler - from simli import SimliClient, SimliConfig except ModuleNotFoundError as e: logger.error(f"Exception: {e}") diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index ff2b7fb87..b701af00d 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -7,24 +7,24 @@ """This module implements Tavus as a sink transport layer""" -import aiohttp import base64 +import aiohttp +from loguru import logger + +from pipecat.audio.utils import resample_audio from pipecat.frames.frames import ( + CancelFrame, + EndFrame, Frame, - TTSAudioRawFrame, + StartInterruptionFrame, TransportMessageUrgentFrame, + TTSAudioRawFrame, TTSStartedFrame, TTSStoppedFrame, - StartInterruptionFrame, - EndFrame, - CancelFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import AIService -from pipecat.audio.utils import resample_audio - -from loguru import logger class TavusVideoService(AIService): diff --git a/src/pipecat/services/to_be_updated/cloudflare_ai_service.py b/src/pipecat/services/to_be_updated/cloudflare_ai_service.py index 1329f9c79..ff637ff1a 100644 --- a/src/pipecat/services/to_be_updated/cloudflare_ai_service.py +++ b/src/pipecat/services/to_be_updated/cloudflare_ai_service.py @@ -1,5 +1,6 @@ -import requests import os + +import requests from services.ai_service import AIService # Note that Cloudflare's AI workers are still in beta. diff --git a/src/pipecat/services/to_be_updated/google_ai_service.py b/src/pipecat/services/to_be_updated/google_ai_service.py index 25668ca0a..3ca688750 100644 --- a/src/pipecat/services/to_be_updated/google_ai_service.py +++ b/src/pipecat/services/to_be_updated/google_ai_service.py @@ -1,11 +1,12 @@ -from services.ai_service import AIService -import openai import os +import openai + # To use Google Cloud's AI products, you'll need to install Google Cloud # CLI and enable the TTS and in your project: # https://cloud.google.com/sdk/docs/install from google.cloud import texttospeech +from services.ai_service import AIService class GoogleAIService(AIService): diff --git a/src/pipecat/services/to_be_updated/mock_ai_service.py b/src/pipecat/services/to_be_updated/mock_ai_service.py index dc200f622..0825cde33 100644 --- a/src/pipecat/services/to_be_updated/mock_ai_service.py +++ b/src/pipecat/services/to_be_updated/mock_ai_service.py @@ -1,6 +1,7 @@ import io -import requests import time + +import requests from PIL import Image from services.ai_service import AIService diff --git a/src/pipecat/services/whisper.py b/src/pipecat/services/whisper.py index a4635c6cb..266ff2b84 100644 --- a/src/pipecat/services/whisper.py +++ b/src/pipecat/services/whisper.py @@ -7,18 +7,16 @@ """This module implements Whisper transcription with a locally-downloaded model.""" import asyncio - from enum import Enum from typing import AsyncGenerator import numpy as np +from loguru import logger from pipecat.frames.frames import ErrorFrame, Frame, TranscriptionFrame from pipecat.services.ai_services import SegmentedSTTService from pipecat.utils.time import time_now_iso8601 -from loguru import logger - try: from faster_whisper import WhisperModel except ModuleNotFoundError as e: diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 573b67717..fbe337d6d 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -29,9 +29,9 @@ from pipecat.frames.frames import ( StartInterruptionFrame, StopInterruptionFrame, SystemFrame, - TTSAudioRawFrame, TransportMessageFrame, TransportMessageUrgentFrame, + TTSAudioRawFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.transports.base_transport import TransportParams diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index 91f95fbbc..27f6cab75 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -6,20 +6,17 @@ import asyncio import inspect - from abc import ABC, abstractmethod from typing import Optional -from pydantic import ConfigDict -from pydantic import BaseModel +from loguru import logger +from pydantic import BaseModel, ConfigDict from pipecat.audio.filters.base_audio_filter import BaseAudioFilter from pipecat.audio.mixers.base_audio_mixer import BaseAudioMixer from pipecat.audio.vad.vad_analyzer import VADAnalyzer from pipecat.processors.frame_processor import FrameProcessor -from loguru import logger - class TransportParams(BaseModel): model_config = ConfigDict(arbitrary_types_allowed=True) diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index e1ccefec2..d62c6bd48 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -5,17 +5,16 @@ # import asyncio - from concurrent.futures import ThreadPoolExecutor +from loguru import logger + from pipecat.frames.frames import InputAudioRawFrame, StartFrame from pipecat.processors.frame_processor import FrameProcessor from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: import pyaudio except ModuleNotFoundError as e: diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index ed7cdbea6..e720f3907 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -5,19 +5,17 @@ # import asyncio - +import tkinter as tk from concurrent.futures import ThreadPoolExecutor import numpy as np -import tkinter as tk +from loguru import logger from pipecat.frames.frames import InputAudioRawFrame, OutputImageRawFrame, StartFrame from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: import pyaudio except ModuleNotFoundError as e: diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index cd21fbe0d..e7ac8b5a5 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -10,8 +10,9 @@ import io import time import typing import wave - from typing import Awaitable, Callable + +from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( @@ -27,8 +28,6 @@ from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: from fastapi import WebSocket from starlette.websockets import WebSocketState diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index ce9b9614d..beac5a3ee 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -8,8 +8,9 @@ import asyncio import io import time import wave - from typing import Awaitable, Callable + +from loguru import logger from pydantic import BaseModel from pipecat.frames.frames import ( @@ -28,8 +29,6 @@ from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: import websockets except ModuleNotFoundError as e: diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index ded16182f..21a91f664 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -8,6 +8,7 @@ import asyncio from dataclasses import dataclass from typing import Any, Awaitable, Callable, List +from loguru import logger from pydantic import BaseModel from pipecat.audio.utils import resample_audio @@ -28,8 +29,6 @@ from pipecat.transports.base_input import BaseInputTransport from pipecat.transports.base_output import BaseOutputTransport from pipecat.transports.base_transport import BaseTransport, TransportParams -from loguru import logger - try: from livekit import rtc from tenacity import retry, stop_after_attempt, wait_exponential diff --git a/src/pipecat/utils/test_frame_processor.py b/src/pipecat/utils/test_frame_processor.py index e46bae7ad..fde476007 100644 --- a/src/pipecat/utils/test_frame_processor.py +++ b/src/pipecat/utils/test_frame_processor.py @@ -1,4 +1,5 @@ from typing import List + from pipecat.processors.frame_processor import FrameProcessor diff --git a/tests/integration/integration_azure_llm.py b/tests/integration/integration_azure_llm.py index 5a2b68c37..8e49e9d04 100644 --- a/tests/integration/integration_azure_llm.py +++ b/tests/integration/integration_azure_llm.py @@ -1,17 +1,17 @@ -import unittest - import asyncio import os +import unittest + +from openai.types.chat import ( + ChatCompletionSystemMessageParam, +) + from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, ) from pipecat.services.azure import AzureLLMService -from openai.types.chat import ( - ChatCompletionSystemMessageParam, -) - if __name__ == "__main__": @unittest.skip("Skip azure integration test") diff --git a/tests/integration/integration_ollama_llm.py b/tests/integration/integration_ollama_llm.py index ced24ed68..085500cb8 100644 --- a/tests/integration/integration_ollama_llm.py +++ b/tests/integration/integration_ollama_llm.py @@ -1,14 +1,14 @@ -import unittest - import asyncio -from pipecat.processors.aggregators.openai_llm_context import ( - OpenAILLMContext, - OpenAILLMContextFrame, -) +import unittest from openai.types.chat import ( ChatCompletionSystemMessageParam, ) + +from pipecat.processors.aggregators.openai_llm_context import ( + OpenAILLMContext, + OpenAILLMContextFrame, +) from pipecat.services.ollama import OLLamaLLMService if __name__ == "__main__": diff --git a/tests/integration/integration_openai_llm.py b/tests/integration/integration_openai_llm.py index 164dcba8d..c788936a1 100644 --- a/tests/integration/integration_openai_llm.py +++ b/tests/integration/integration_openai_llm.py @@ -3,17 +3,16 @@ import json import os from typing import List -from pipecat.services.openai import OpenAILLMContextFrame, OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.frames.frames import LLMFullResponseStartFrame, LLMFullResponseEndFrame, TextFrame -from pipecat.utils.test_frame_processor import TestFrameProcessor from openai.types.chat import ( ChatCompletionSystemMessageParam, ChatCompletionToolParam, ChatCompletionUserMessageParam, ) -from pipecat.services.openai import OpenAILLMService +from pipecat.frames.frames import LLMFullResponseEndFrame, LLMFullResponseStartFrame, TextFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.openai import OpenAILLMContext, OpenAILLMContextFrame, OpenAILLMService +from pipecat.utils.test_frame_processor import TestFrameProcessor tools = [ ChatCompletionToolParam( diff --git a/tests/test_aggregators.py b/tests/test_aggregators.py index 76834183c..dcf27ad6e 100644 --- a/tests/test_aggregators.py +++ b/tests/test_aggregators.py @@ -3,23 +3,20 @@ import doctest import functools import unittest -from pipecat.processors.aggregators.gated import GatedAggregator -from pipecat.processors.aggregators.sentence import SentenceAggregator -from pipecat.processors.text_transformer import StatelessTextTransformer - -from pipecat.pipeline.parallel_pipeline import ParallelPipeline - from pipecat.frames.frames import ( AudioRawFrame, EndFrame, + Frame, ImageRawFrame, LLMFullResponseEndFrame, LLMFullResponseStartFrame, - Frame, TextFrame, ) - +from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators.gated import GatedAggregator +from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.text_transformer import StatelessTextTransformer class TestDailyFrameAggregators(unittest.IsolatedAsyncioTestCase): diff --git a/tests/test_ai_services.py b/tests/test_ai_services.py index 975f7e20c..13aa20467 100644 --- a/tests/test_ai_services.py +++ b/tests/test_ai_services.py @@ -1,9 +1,8 @@ import unittest - from typing import AsyncGenerator -from pipecat.services.ai_services import AIService, match_endofsentence from pipecat.frames.frames import EndFrame, Frame, TextFrame +from pipecat.services.ai_services import AIService, match_endofsentence class SimpleAIService(AIService): diff --git a/tests/test_langchain.py b/tests/test_langchain.py index d30d213bd..bed3f907a 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -6,6 +6,9 @@ import unittest +from langchain.prompts import ChatPromptTemplate +from langchain_core.language_models import FakeStreamingListLLM + from pipecat.frames.frames import ( EndFrame, LLMFullResponseEndFrame, @@ -25,9 +28,6 @@ from pipecat.processors.aggregators.llm_response import ( from pipecat.processors.frame_processor import FrameProcessor from pipecat.processors.frameworks.langchain import LangchainProcessor -from langchain.prompts import ChatPromptTemplate -from langchain_core.language_models import FakeStreamingListLLM - class TestLangchain(unittest.IsolatedAsyncioTestCase): class MockProcessor(FrameProcessor): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ba82974bc..7d703d5ed 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,12 +2,11 @@ import asyncio import unittest from unittest.mock import Mock -from pipecat.processors.aggregators.sentence import SentenceAggregator -from pipecat.processors.text_transformer import StatelessTextTransformer -from pipecat.processors.frame_processor import FrameProcessor from pipecat.frames.frames import EndFrame, TextFrame - from pipecat.pipeline.pipeline import Pipeline +from pipecat.processors.aggregators.sentence import SentenceAggregator +from pipecat.processors.frame_processor import FrameProcessor +from pipecat.processors.text_transformer import StatelessTextTransformer class TestDailyPipeline(unittest.IsolatedAsyncioTestCase): From 5c57cccea3006f03866f56399350009aed9e2d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 11:29:28 -0800 Subject: [PATCH 41/90] github: run ruff check import linter --- .github/workflows/format.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml index 5866f881b..444e24338 100644 --- a/.github/workflows/format.yaml +++ b/.github/workflows/format.yaml @@ -35,7 +35,12 @@ jobs: python -m pip install --upgrade pip pip install -r dev-requirements.txt - name: Ruff formatter - id: ruff + id: ruff-format run: | source .venv/bin/activate ruff format --diff + - name: Ruff import linter + id: ruff-check + run: | + source .venv/bin/activate + ruff check --select I From b9ca667d3104f1517f3a03b69c12e7bbd5fc2974 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 11:40:43 -0800 Subject: [PATCH 42/90] pyproject: use tool.ruff.lint sections --- pyproject.toml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1a43b1b89..8b8026a8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,9 +89,11 @@ fallback_version = "0.0.0-dev" exclude = ["*_pb2.py"] line-length = 100 +[tool.ruff.lint] select = [ "D", # Docstring rules + "I", # Import rules ] -[tool.ruff.pydocstyle] -convention = "google" \ No newline at end of file +[tool.ruff.lint.pydocstyle] +convention = "google" From 5bfcac1f5ca383040ccfe4a34e51334b7be3404f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 16:02:33 -0800 Subject: [PATCH 43/90] transports: call parent stop() before disconnecting This rollbacks a previous change https://github.com/pipecat-ai/pipecat/pull/855 which was trying to fix an issue in the wrong way. The reasoning behind this fix is that the parent class might be sending audio or messages (through the subclass) and if we disconnect before all the data is sent we will run into incomplete audio or even errors. Therefore, we first make sure the parent tasks stop and then it will be safe to disconnect. --- .../transports/network/websocket_server.py | 4 +-- src/pipecat/transports/services/daily.py | 26 +++++++++---------- src/pipecat/transports/services/livekit.py | 8 +++--- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index beac5a3ee..58d104038 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -72,14 +72,14 @@ class WebsocketServerInputTransport(BaseInputTransport): self._server_task = self.get_event_loop().create_task(self._server_task_handler()) async def stop(self, frame: EndFrame): + await super().stop(frame) self._stop_server_event.set() await self._server_task - await super().stop(frame) async def cancel(self, frame: CancelFrame): + await super().cancel(frame) self._stop_server_event.set() await self._server_task - await super().cancel(frame) async def _server_task_handler(self): logger.info(f"Starting websocket server on {self._host}:{self._port}") diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 44c9f37ee..7456ef816 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -694,17 +694,8 @@ class DailyInputTransport(BaseInputTransport): self._audio_in_task = self.get_event_loop().create_task(self._audio_in_task_handler()) async def stop(self, frame: EndFrame): - # Leave the room. - await self._client.leave() - # Stop audio thread. - if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): - self._audio_in_task.cancel() - await self._audio_in_task - self._audio_in_task = None # Parent stop. await super().stop(frame) - - async def cancel(self, frame: CancelFrame): # Leave the room. await self._client.leave() # Stop audio thread. @@ -712,8 +703,17 @@ class DailyInputTransport(BaseInputTransport): self._audio_in_task.cancel() await self._audio_in_task self._audio_in_task = None + + async def cancel(self, frame: CancelFrame): # Parent stop. await super().cancel(frame) + # Leave the room. + await self._client.leave() + # Stop audio thread. + if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): + self._audio_in_task.cancel() + await self._audio_in_task + self._audio_in_task = None async def cleanup(self): await super().cleanup() @@ -817,16 +817,16 @@ class DailyOutputTransport(BaseOutputTransport): await self._client.join() async def stop(self, frame: EndFrame): - # Leave the room. - await self._client.leave() # Parent stop. await super().stop(frame) - - async def cancel(self, frame: CancelFrame): # Leave the room. await self._client.leave() + + async def cancel(self, frame: CancelFrame): # Parent stop. await super().cancel(frame) + # Leave the room. + await self._client.leave() async def cleanup(self): await super().cleanup() diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 21a91f664..81a0ffdd1 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -323,19 +323,19 @@ class LiveKitInputTransport(BaseInputTransport): logger.info("LiveKitInputTransport started") async def stop(self, frame: EndFrame): + await super().stop(frame) await self._client.disconnect() if self._audio_in_task: self._audio_in_task.cancel() await self._audio_in_task - await super().stop(frame) logger.info("LiveKitInputTransport stopped") async def cancel(self, frame: CancelFrame): + await super().cancel(frame) await self._client.disconnect() if self._audio_in_task and (self._params.audio_in_enabled or self._params.vad_enabled): self._audio_in_task.cancel() await self._audio_in_task - await super().cancel(frame) def vad_analyzer(self) -> VADAnalyzer | None: return self._vad_analyzer @@ -397,13 +397,13 @@ class LiveKitOutputTransport(BaseOutputTransport): logger.info("LiveKitOutputTransport started") async def stop(self, frame: EndFrame): - await self._client.disconnect() await super().stop(frame) + await self._client.disconnect() logger.info("LiveKitOutputTransport stopped") async def cancel(self, frame: CancelFrame): - await self._client.disconnect() await super().cancel(frame) + await self._client.disconnect() async def send_message(self, frame: TransportMessageFrame | TransportMessageUrgentFrame): if isinstance(frame, (LiveKitTransportMessageFrame, LiveKitTransportMessageUrgentFrame)): From 2dfdceb9e6ab820fd400c2347b4ea362cbae9d10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 16:12:49 -0800 Subject: [PATCH 44/90] processors(filters): allow passing EndFrame --- src/pipecat/processors/filters/frame_filter.py | 4 ++-- src/pipecat/processors/filters/function_filter.py | 7 ++++--- src/pipecat/processors/filters/null_filter.py | 13 ++++++++++++- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py index 11f2e601a..674670163 100644 --- a/src/pipecat/processors/filters/frame_filter.py +++ b/src/pipecat/processors/filters/frame_filter.py @@ -6,7 +6,7 @@ from typing import Tuple, Type -from pipecat.frames.frames import ControlFrame, Frame, SystemFrame +from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -23,7 +23,7 @@ class FrameFilter(FrameProcessor): if isinstance(frame, self._types): return True - return isinstance(frame, ControlFrame) or isinstance(frame, SystemFrame) + return isinstance(frame, (EndFrame, SystemFrame)) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index e38cea3e0..1d06be02c 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -6,7 +6,7 @@ from typing import Awaitable, Callable -from pipecat.frames.frames import Frame, SystemFrame +from pipecat.frames.frames import EndFrame, Frame, SystemFrame from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -24,9 +24,10 @@ class FunctionFilter(FrameProcessor): # Frame processor # - # Ignore system frames and frames that are not following the direction of this gate + # Ignore system frames, end frames and frames that are not following the + # direction of this gate def _should_passthrough_frame(self, frame, direction): - return isinstance(frame, SystemFrame) or direction != self._direction + return isinstance(frame, (SystemFrame, EndFrame)) or direction != self._direction async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) diff --git a/src/pipecat/processors/filters/null_filter.py b/src/pipecat/processors/filters/null_filter.py index 7e9ca6725..219cc149c 100644 --- a/src/pipecat/processors/filters/null_filter.py +++ b/src/pipecat/processors/filters/null_filter.py @@ -4,7 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from pipecat.processors.frame_processor import FrameProcessor +from pipecat.frames.frames import EndFrame, Frame, SystemFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class NullFilter(FrameProcessor): @@ -12,3 +13,13 @@ class NullFilter(FrameProcessor): def __init__(self, **kwargs): super().__init__(**kwargs) + + # + # Frame processor + # + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, (SystemFrame, EndFrame)): + await self.push_frame(frame, direction) From 42bea578e85392569b3654ffa963814feb46b505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 16:21:02 -0800 Subject: [PATCH 45/90] pipeline(parallel): wait for slowest endframe If we are sending an EndFrame and a ParallelPipeline has multiple pipelines we want to wait before pushing the EndFrame downstream until the slowest pipeline is finished. Otherwise, we could be disconnecting from the transport too early. --- CHANGELOG.md | 3 + src/pipecat/pipeline/parallel_pipeline.py | 112 ++++++++++++++++------ 2 files changed, 84 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e069fc7..1ed377339 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue that would cause `ParallelPipeline` to handle `EndFrame` + incorrectly causing the main pipeline to not terminate or terminate too early. + - Fixed an audio stuttering issue in `FastPitchTTSService`. - Fixed a `BaseOutputTransport` issue that was causing non-audio frames being diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index 323f7ed24..b39258782 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -6,11 +6,18 @@ import asyncio from itertools import chain -from typing import Awaitable, Callable, List +from typing import Awaitable, Callable, Dict, List from loguru import logger -from pipecat.frames.frames import CancelFrame, EndFrame, Frame, StartFrame, SystemFrame +from pipecat.frames.frames import ( + CancelFrame, + EndFrame, + Frame, + StartFrame, + StartInterruptionFrame, + SystemFrame, +) from pipecat.pipeline.base_pipeline import BasePipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -72,11 +79,10 @@ class ParallelPipeline(BasePipeline): self._sources = [] self._sinks = [] self._seen_ids = set() + self._endframe_counter: Dict[int, int] = {} self._up_queue = asyncio.Queue() self._down_queue = asyncio.Queue() - self._up_task: asyncio.Task | None = None - self._down_task: asyncio.Task | None = None self._pipelines = [] @@ -111,18 +117,19 @@ class ParallelPipeline(BasePipeline): # async def cleanup(self): + await asyncio.gather(*[s.cleanup() for s in self._sources]) await asyncio.gather(*[p.cleanup() for p in self._pipelines]) - - async def _start_tasks(self): - loop = self.get_event_loop() - self._up_task = loop.create_task(self._process_up_queue()) - self._down_task = loop.create_task(self._process_down_queue()) + await asyncio.gather(*[s.cleanup() for s in self._sinks]) async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, StartFrame): - await self._start_tasks() + await self._start() + elif isinstance(frame, EndFrame): + self._endframe_counter[frame.id] = len(self._pipelines) + elif isinstance(frame, CancelFrame): + await self._cancel() if direction == FrameDirection.UPSTREAM: # If we get an upstream frame we process it in each sink. @@ -131,16 +138,45 @@ class ParallelPipeline(BasePipeline): # If we get a downstream frame we process it in each source. await asyncio.gather(*[s.queue_frame(frame, direction) for s in self._sources]) - # If we get an EndFrame we stop our queue processing tasks and wait on - # all the pipelines to finish. - if isinstance(frame, (CancelFrame, EndFrame)): - # Use None to indicate when queues should be done processing. - await self._up_queue.put(None) - await self._down_queue.put(None) - if self._up_task: - await self._up_task - if self._down_task: - await self._down_task + # Handle interruptions after everything has been cancelled. + if isinstance(frame, StartInterruptionFrame): + await self._handle_interruption() + # Wait for tasks to finish. + elif isinstance(frame, EndFrame): + await self._stop() + + async def _start(self): + await self._create_tasks() + + async def _stop(self): + # The up task doesn't receive an EndFrame, so we just cancel it. + self._up_task.cancel() + await self._up_task + # The down tasks waits for the last EndFrame send by the internal + # pipelines. + await self._down_task + + async def _cancel(self): + self._up_task.cancel() + await self._up_task + self._down_task.cancel() + await self._down_task + + async def _create_tasks(self): + loop = self.get_event_loop() + self._up_task = loop.create_task(self._process_up_queue()) + self._down_task = loop.create_task(self._process_down_queue()) + + async def _drain_queues(self): + while not self._up_queue.empty: + await self._up_queue.get() + while not self._down_queue.empty: + await self._down_queue.get() + + async def _handle_interruption(self): + await self._cancel() + await self._drain_queues() + await self._create_tasks() async def _parallel_push_frame(self, frame: Frame, direction: FrameDirection): if frame.id not in self._seen_ids: @@ -148,19 +184,33 @@ class ParallelPipeline(BasePipeline): await self.push_frame(frame, direction) async def _process_up_queue(self): - running = True - while running: - frame = await self._up_queue.get() - if frame: + while True: + try: + frame = await self._up_queue.get() await self._parallel_push_frame(frame, FrameDirection.UPSTREAM) - running = frame is not None - self._up_queue.task_done() + self._up_queue.task_done() + except asyncio.CancelledError: + break async def _process_down_queue(self): running = True while running: - frame = await self._down_queue.get() - if frame: - await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM) - running = frame is not None - self._down_queue.task_done() + try: + frame = await self._down_queue.get() + + endframe_counter = self._endframe_counter.get(frame.id, 0) + + # If we have a counter, decrement it. + if endframe_counter > 0: + self._endframe_counter[frame.id] -= 1 + endframe_counter = self._endframe_counter[frame.id] + + # If we don't have a counter or we reached 0, push the frame. + if endframe_counter == 0: + await self._parallel_push_frame(frame, FrameDirection.DOWNSTREAM) + + running = not (endframe_counter == 0 and isinstance(frame, EndFrame)) + + self._down_queue.task_done() + except asyncio.CancelledError: + break From 7322badbe78645b0cf0b3580aa9c80f8437c4d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 18:14:04 -0800 Subject: [PATCH 46/90] audio(koala): add new audio filter KoalaFilter --- CHANGELOG.md | 4 ++ pyproject.toml | 1 + src/pipecat/audio/filters/koala_filter.py | 75 +++++++++++++++++++++++ 3 files changed, 80 insertions(+) create mode 100644 src/pipecat/audio/filters/koala_filter.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e069fc7..200875dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `KoalaFilter` which implement on device noise reduction using Koala + Noise Suppression. + (see https://picovoice.ai/platform/koala/) + - Pipecat now supports Python 3.13. We had a dependency on the `audioop` package which was deprecated and now removed on Python 3.13. We are now using `audioop-lts` (https://github.com/AbstractUmbra/audioop) to provide the same diff --git a/pyproject.toml b/pyproject.toml index 8b8026a8e..4cf1de72c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ groq = [ "openai~=1.57.2" ] gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.57.2" ] krisp = [ "pipecat-ai-krisp~=0.3.0" ] +koala = [ "pvkoala~=2.0.2" ] langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1" ] lmnt = [ "lmnt~=1.1.4" ] diff --git a/src/pipecat/audio/filters/koala_filter.py b/src/pipecat/audio/filters/koala_filter.py new file mode 100644 index 000000000..416e4e9fb --- /dev/null +++ b/src/pipecat/audio/filters/koala_filter.py @@ -0,0 +1,75 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import Sequence + +import numpy as np +from loguru import logger + +from pipecat.audio.filters.base_audio_filter import BaseAudioFilter +from pipecat.frames.frames import FilterControlFrame, FilterEnableFrame + +try: + import pvkoala +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use the Koala filter, you need to `pip install pipecat-ai[koala]`.") + raise Exception(f"Missing module: {e}") + + +class KoalaFilter(BaseAudioFilter): + """This is an audio filter that uses Koala Noise Suppression (from + PicoVoice). + + """ + + def __init__(self, *, access_key: str) -> None: + self._access_key = access_key + + self._filtering = True + self._sample_rate = 0 + self._koala = pvkoala.create(access_key=f"{self._access_key}") + self._koala_ready = True + self._audio_buffer = bytearray() + + async def start(self, sample_rate: int): + self._sample_rate = sample_rate + if self._sample_rate != self._koala.sample_rate: + logger.warning( + f"Koala filter needs sample rate {self._koala.sample_rate} (got {self._sample_rate})" + ) + self._koala_ready = False + + async def stop(self): + self._koala.reset() + + async def process_frame(self, frame: FilterControlFrame): + if isinstance(frame, FilterEnableFrame): + self._filtering = frame.enable + + async def filter(self, audio: bytes) -> bytes: + if not self._koala_ready or not self._filtering: + return audio + + self._audio_buffer.extend(audio) + + filtered_data: Sequence[int] = [] + + num_frames = len(self._audio_buffer) // 2 + while num_frames >= self._koala.frame_length: + # Grab the number of frames required by Koala. + num_bytes = self._koala.frame_length * 2 + audio = bytes(self._audio_buffer[:num_bytes]) + # Process audio + data = np.frombuffer(audio, dtype=np.int16).tolist() + filtered_data += self._koala.process(data) + # Adjust audio buffer and check again + self._audio_buffer = self._audio_buffer[num_bytes:] + num_frames = len(self._audio_buffer) // 2 + + filtered = np.array(filtered_data, dtype=np.int16).tobytes() + + return filtered From 55879bf36561052eccbb0d9447c2dabfc01ff5f0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 13 Dec 2024 15:38:59 -0500 Subject: [PATCH 47/90] Add TranscriptionProcessor --- .../07a-interruptible-anthropic.py | 42 ++++- .../28a-transcription-update-openai.py | 128 +++++++++++++++ .../28b-transcription-update-anthropic.py | 128 +++++++++++++++ .../28c-transcription-update-gemini.py | 138 ++++++++++++++++ src/pipecat/frames/frames.py | 36 ++++- .../processors/transcript_processor.py | 150 ++++++++++++++++++ 6 files changed, 613 insertions(+), 9 deletions(-) create mode 100644 examples/foundational/28a-transcription-update-openai.py create mode 100644 examples/foundational/28b-transcription-update-anthropic.py create mode 100644 examples/foundational/28c-transcription-update-gemini.py create mode 100644 src/pipecat/processors/transcript_processor.py diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index e7e680eab..25a301269 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -7,6 +7,7 @@ import asyncio import os import sys +from typing import List import aiohttp from dotenv import load_dotenv @@ -14,12 +15,13 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame +from pipecat.frames.frames import Frame, LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.services.anthropic import AnthropicLLMService +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.anthropic import AnthropicLLMContext, AnthropicLLMService from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -29,6 +31,28 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") +class TestAnthropicLLMService(AnthropicLLMService): + async def process_frame(self, frame: Frame, direction: FrameDirection): + if isinstance(frame, LLMMessagesFrame): + logger.info("Original OpenAI format messages:") + logger.info(frame.messages) + + # Convert to Anthropic format + context = AnthropicLLMContext.from_messages(frame.messages) + logger.info("Converted to Anthropic format:") + logger.info(context.messages) + + # Convert back to OpenAI format + openai_messages = [] + for msg in context.messages: + converted = context.to_standard_messages(msg) + openai_messages.extend(converted) + logger.info("Converted back to OpenAI format:") + logger.info(openai_messages) + + await super().process_frame(frame, direction) + + async def main(): async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) @@ -50,18 +74,24 @@ async def main(): voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) - llm = AnthropicLLMService( + llm = TestAnthropicLLMService( api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-opus-20240229" ) - # todo: think more about how to handle system prompts in a more general way. OpenAI, - # Google, and Anthropic all have slightly different approaches to providing a system - # prompt. + # Test messages including various formats messages = [ { "role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", }, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Hello! How can I help you today?"}, + {"type": "text", "text": "I'm ready to assist."}, + ], + }, + {"role": "user", "content": "Hi there!"}, ] context = OpenAILLMContext(messages) diff --git a/examples/foundational/28a-transcription-update-openai.py b/examples/foundational/28a-transcription-update-openai.py new file mode 100644 index 000000000..ec103ff82 --- /dev/null +++ b/examples/foundational/28a-transcription-update-openai.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys +from typing import List + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame, TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +class TranscriptHandler: + """Simple handler to demonstrate transcript processing.""" + + def __init__(self): + self.messages: List[TranscriptionMessage] = [] + + async def on_transcript_update( + self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame + ): + """Handle new transcript messages.""" + self.messages.extend(frame.messages) + + # Log the new messages + logger.info("New transcript messages:") + for msg in frame.messages: + logger.info(f"{msg.role}: {msg.content}") + + # Log the full transcript + logger.info("Full transcript:") + for msg in self.messages: + logger.info(f"{msg.role}: {msg.content}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = OpenAILLMService( + api_key=os.getenv("OPENAI_API_KEY"), + model="gpt-4o", + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript_processor = TranscriptProcessor() + transcript_handler = TranscriptHandler() + + # Register event handler for transcript updates + @transcript_processor.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + transcript_processor, # Process transcripts + ] + ) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/28b-transcription-update-anthropic.py b/examples/foundational/28b-transcription-update-anthropic.py new file mode 100644 index 000000000..23ee93a21 --- /dev/null +++ b/examples/foundational/28b-transcription-update-anthropic.py @@ -0,0 +1,128 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys +from typing import List + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame, TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.anthropic import AnthropicLLMService +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +class TranscriptHandler: + """Simple handler to demonstrate transcript processing.""" + + def __init__(self): + self.messages: List[TranscriptionMessage] = [] + + async def on_transcript_update( + self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame + ): + """Handle new transcript messages.""" + self.messages.extend(frame.messages) + + # Log the new messages + logger.info("New transcript messages:") + for msg in frame.messages: + logger.info(f"{msg.role}: {msg.content}") + + # Log the full transcript + logger.info("Full transcript:") + for msg in self.messages: + logger.info(f"{msg.role}: {msg.content}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-5-sonnet-20241022" + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", + }, + {"role": "user", "content": "Say hello."}, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript_processor = TranscriptProcessor() + transcript_handler = TranscriptHandler() + + # Register event handler for transcript updates + @transcript_processor.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + transcript_processor, # Process transcripts + ] + ) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/foundational/28c-transcription-update-gemini.py b/examples/foundational/28c-transcription-update-gemini.py new file mode 100644 index 000000000..27291a7c9 --- /dev/null +++ b/examples/foundational/28c-transcription-update-gemini.py @@ -0,0 +1,138 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys +from typing import List + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.transcript_processor import TranscriptProcessor +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.google import GoogleLLMService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +class TranscriptHandler: + """Simple handler to demonstrate transcript processing.""" + + def __init__(self): + self.messages: List[TranscriptionMessage] = [] + + async def on_transcript_update( + self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame + ): + """Handle new transcript messages.""" + self.messages.extend(frame.messages) + + # Log the new messages + logger.info("New transcript messages:") + for msg in frame.messages: + logger.info(f"{msg.role}: {msg.content}") + + # Log the full transcript + logger.info("Full transcript:") + for msg in self.messages: + logger.info(f"{msg.role}: {msg.content}") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = GoogleLLMService( + model="models/gemini-2.0-flash-exp", + # model="gemini-exp-1114", + api_key=os.getenv("GOOGLE_API_KEY"), + ) + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way.", + }, + {"role": "user", "content": "Say hello."}, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + # Create transcript processor and handler + transcript_processor = TranscriptProcessor() + transcript_handler = TranscriptHandler() + + # Register event handler for transcript updates + @transcript_processor.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + transcript_processor, + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index d3792f537..f74d30371 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -5,7 +5,7 @@ # from dataclasses import dataclass, field -from typing import Any, List, Mapping, Optional, Tuple +from typing import Any, List, Literal, Mapping, Optional, Tuple, TypeAlias from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.clocks.base_clock import BaseClock @@ -195,7 +195,8 @@ class TranscriptionFrame(TextFrame): @dataclass class InterimTranscriptionFrame(TextFrame): """A text frame with interim transcription-specific data. Will be placed in - the transport's receive queue when a participant speaks.""" + the transport's receive queue when a participant speaks. + """ text: str user_id: str @@ -206,6 +207,34 @@ class InterimTranscriptionFrame(TextFrame): return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})" +@dataclass +class TranscriptionMessage: + """A message in a conversation transcript containing the role and content. + + Messages are in standard format with roles normalized to user/assistant. + """ + + role: Literal["user", "assistant"] + content: str + timestamp: str | None = None + + +@dataclass +class TranscriptionUpdateFrame(DataFrame): + """A frame containing new messages added to the conversation transcript. + + This frame is emitted when new messages are added to the conversation history, + containing only the newly added messages rather than the full transcript. + Messages have normalized roles (user/assistant) regardless of the LLM service used. + """ + + messages: List[TranscriptionMessage] + + def __str__(self): + pts = format_pts(self.pts) + return f"{self.name}(pts: {pts}, messages: {len(self.messages)})" + + @dataclass class LLMMessagesFrame(DataFrame): """A frame containing a list of LLM messages. Used to signal that an LLM @@ -546,7 +575,8 @@ class EndFrame(ControlFrame): @dataclass class LLMFullResponseStartFrame(ControlFrame): """Used to indicate the beginning of an LLM response. Following by one or - more TextFrame and a final LLMFullResponseEndFrame.""" + more TextFrame and a final LLMFullResponseEndFrame. + """ pass diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py new file mode 100644 index 000000000..97173f967 --- /dev/null +++ b/src/pipecat/processors/transcript_processor.py @@ -0,0 +1,150 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from typing import List + +from loguru import logger + +from pipecat.frames.frames import ( + ErrorFrame, + Frame, + TranscriptionMessage, + TranscriptionUpdateFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor + + +class TranscriptProcessor(FrameProcessor): + """Processes LLM context frames to generate conversation transcripts. + + This processor monitors OpenAILLMContextFrame frames and extracts conversation + content, filtering out system messages and function calls. When new messages + are detected, it emits a TranscriptionUpdateFrame containing only the new + messages. + + Each LLM context (OpenAI, Anthropic, Google) provides conversion to the standard format: + [ + { + "role": "user", + "content": [{"type": "text", "text": "Hi, how are you?"}] + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "Great! And you?"}] + } + ] + + Events: + on_transcript_update: Emitted when new transcript messages are available. + Args: TranscriptionUpdateFrame containing new messages. + + Example: + ```python + transcript_processor = TranscriptProcessor() + + @transcript_processor.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + for msg in frame.messages: + print(f"{msg.role}: {msg.content}") + ``` + """ + + def __init__(self, **kwargs): + """Initialize the transcript processor. + + Args: + **kwargs: Additional arguments passed to FrameProcessor + """ + super().__init__(**kwargs) + self._processed_messages: List[TranscriptionMessage] = [] + self._register_event_handler("on_transcript_update") + + def _extract_messages(self, messages: List[dict]) -> List[TranscriptionMessage]: + """Extract conversation messages from standard format. + + Args: + messages: List of messages in standard format with structured content + + Returns: + List[TranscriptionMessage]: Normalized conversation messages + """ + result = [] + for msg in messages: + # Only process user and assistant messages + if msg["role"] not in ("user", "assistant"): + continue + + content = msg.get("content", []) + if isinstance(content, list): + # Extract text from structured content + text_parts = [] + for part in content: + if isinstance(part, dict) and part.get("type") == "text": + text_parts.append(part["text"]) + + if text_parts: + result.append( + TranscriptionMessage(role=msg["role"], content=" ".join(text_parts)) + ) + + return result + + def _find_new_messages(self, current: List[TranscriptionMessage]) -> List[TranscriptionMessage]: + """Find messages in current that aren't in self._processed_messages. + + Args: + current: List of current messages + + Returns: + List[TranscriptionMessage]: New messages not yet processed + """ + if not self._processed_messages: + return current + + processed_len = len(self._processed_messages) + if len(current) <= processed_len: + return [] + + return current[processed_len:] + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames, watching for OpenAILLMContextFrame. + + Args: + frame: The frame to process + direction: Frame processing direction + + Raises: + ErrorFrame: If message processing fails + """ + await super().process_frame(frame, direction) + + if isinstance(frame, OpenAILLMContextFrame): + try: + # Convert context messages to standard format + standard_messages = [] + for msg in frame.context.messages: + converted = frame.context.to_standard_messages(msg) + standard_messages.extend(converted) + + # Extract and process messages + current_messages = self._extract_messages(standard_messages) + new_messages = self._find_new_messages(current_messages) + + if new_messages: + # Update state and notify listeners + self._processed_messages.extend(new_messages) + update_frame = TranscriptionUpdateFrame(messages=new_messages) + await self._call_event_handler("on_transcript_update", update_frame) + await self.push_frame(update_frame) + + except Exception as e: + logger.error(f"Error processing transcript in {self}: {e}") + await self.push_error(ErrorFrame(str(e))) + + # Always push the original frame downstream + await self.push_frame(frame, direction) From 4f2aee5fba578862c17caf2c41c239bed6094990 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 14 Dec 2024 09:14:34 -0500 Subject: [PATCH 48/90] Update OpenAI's to_standard_messages to return the verboase message format --- .../aggregators/openai_llm_context.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 6e7474c17..e84584053 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -115,8 +115,25 @@ class OpenAILLMContext: def from_standard_message(self, message): return message - # convert a message in this LLM's format to one or more messages in OpenAI format def to_standard_messages(self, obj) -> list: + """Convert OpenAI message to standard structured format. + + Args: + obj: Message in OpenAI format {"role": "user", "content": "text"} + + Returns: + List containing message with structured content: + [{"role": "user", "content": [{"type": "text", "text": "message"}]}] + """ + # Skip messages without content + if not obj.get("content"): + return [] + + # Convert simple string content to structured format + if isinstance(obj["content"], str): + return [{"role": obj["role"], "content": [{"type": "text", "text": obj["content"]}]}] + + # Return original message if content is already structured return [obj] def get_messages_for_initializing_history(self): From 51b235df4b022db807c1e3365d5f81564403aa6f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 14 Dec 2024 09:22:33 -0500 Subject: [PATCH 49/90] Add docstrings for Google and Anthropic's to_standard_messages and from_standard_message functions --- src/pipecat/services/anthropic.py | 44 +++++++++++++++++++++++++++++++ src/pipecat/services/google.py | 40 ++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index f0c033375..d8f485296 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -378,6 +378,26 @@ class AnthropicLLMContext(OpenAILLMContext): # convert a message in Anthropic format into one or more messages in OpenAI format def to_standard_messages(self, obj): + """Convert Anthropic message format to standard structured format. + + Handles text content and function calls for both user and assistant messages. + + Args: + obj: Message in Anthropic format: + { + "role": "user/assistant", + "content": str | [{"type": "text/tool_use/tool_result", ...}] + } + + Returns: + List of messages in standard format: + [ + { + "role": "user/assistant/tool", + "content": [{"type": "text", "text": str}] + } + ] + """ # todo: image format (?) # tool_use role = obj.get("role") @@ -432,6 +452,30 @@ class AnthropicLLMContext(OpenAILLMContext): return messages def from_standard_message(self, message): + """Convert standard format message to Anthropic format. + + Handles conversion of text content, tool calls, and tool results. + Empty text content is converted to "(empty)". + + Args: + message: Message in standard format: + { + "role": "user/assistant/tool", + "content": str | [{"type": "text", ...}], + "tool_calls": [{"id": str, "function": {"name": str, "arguments": str}}] + } + + Returns: + Message in Anthropic format: + { + "role": "user/assistant", + "content": str | [ + {"type": "text", "text": str} | + {"type": "tool_use", "id": str, "name": str, "input": dict} | + {"type": "tool_result", "tool_use_id": str, "content": str} + ] + } + """ # todo: image messages (?) if message["role"] == "tool": return { diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 5442ee91c..383dde624 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -412,6 +412,25 @@ class GoogleLLMContext(OpenAILLMContext): # self.add_message(message) def from_standard_message(self, message): + """Convert standard format message to Google Content object. + + Handles conversion of text, images, and function calls to Google's format. + System messages are stored separately and return None. + + Args: + message: Message in standard format: + { + "role": "user/assistant/system/tool", + "content": str | [{"type": "text/image_url", ...}] | None, + "tool_calls": [{"function": {"name": str, "arguments": str}}] + } + + Returns: + glm.Content object with: + - role: "user" or "model" (converted from "assistant") + - parts: List[Part] containing text, inline_data, or function calls + Returns None for system messages. + """ role = message["role"] content = message.get("content", []) if role == "system": @@ -461,6 +480,27 @@ class GoogleLLMContext(OpenAILLMContext): return message def to_standard_messages(self, obj) -> list: + """Convert Google Content object to standard structured format. + + Handles text, images, and function calls from Google's Content/Part objects. + + Args: + obj: Google Content object with: + - role: "model" (converted to "assistant") or "user" + - parts: List[Part] containing text, inline_data, or function calls + + Returns: + List of messages in standard format: + [ + { + "role": "user/assistant/tool", + "content": [ + {"type": "text", "text": str} | + {"type": "image_url", "image_url": {"url": str}} + ] + } + ] + """ msg = {"role": obj.role, "content": []} if msg["role"] == "model": msg["role"] = "assistant" From 77aeda36eba644bd4471f98a0fd0d11ba94fbf2f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 14 Dec 2024 09:23:43 -0500 Subject: [PATCH 50/90] Update OpenAI's from_standard_message to convert back to OpenAI's simple format --- .../aggregators/openai_llm_context.py | 33 ++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index e84584053..4adf76de0 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -112,7 +112,38 @@ class OpenAILLMContext: msgs.append(msg) return json.dumps(msgs) - def from_standard_message(self, message): + def from_standard_message(self, message) -> dict: + """Convert standard format message to OpenAI format. + + Converts structured content back to OpenAI's simple string format. + + Args: + message: Message in standard format: + { + "role": "user/assistant", + "content": [{"type": "text", "text": str}] + } + + Returns: + Message in OpenAI format: + { + "role": "user/assistant", + "content": str + } + """ + # If content is already a string, return as-is + if isinstance(message.get("content"), str): + return message + + # Convert structured content to string + if isinstance(message.get("content"), list): + text_parts = [] + for part in message["content"]: + if part.get("type") == "text": + text_parts.append(part["text"]) + + return {"role": message["role"], "content": " ".join(text_parts) if text_parts else ""} + return message def to_standard_messages(self, obj) -> list: From dd2703317ae125d2c1a12653476e49f7d7712f71 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 14 Dec 2024 11:03:08 -0500 Subject: [PATCH 51/90] Add timestamp frames and include timestamps in the transcription event and frame --- .../28a-transcription-update-openai.py | 24 ++++-- .../28b-transcription-update-anthropic.py | 24 ++++-- .../28c-transcription-update-gemini.py | 24 ++++-- src/pipecat/frames/frames.py | 14 +++ .../processors/aggregators/llm_response.py | 6 ++ .../processors/transcript_processor.py | 86 ++++++++++++------- src/pipecat/services/anthropic.py | 7 ++ src/pipecat/services/google.py | 13 +++ src/pipecat/services/openai.py | 7 ++ 9 files changed, 155 insertions(+), 50 deletions(-) diff --git a/examples/foundational/28a-transcription-update-openai.py b/examples/foundational/28a-transcription-update-openai.py index ec103ff82..390343465 100644 --- a/examples/foundational/28a-transcription-update-openai.py +++ b/examples/foundational/28a-transcription-update-openai.py @@ -32,7 +32,10 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptHandler: - """Simple handler to demonstrate transcript processing.""" + """Simple handler to demonstrate transcript processing. + + Maintains a list of conversation messages and logs them with timestamps. + """ def __init__(self): self.messages: List[TranscriptionMessage] = [] @@ -40,18 +43,25 @@ class TranscriptHandler: async def on_transcript_update( self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame ): - """Handle new transcript messages.""" + """Handle new transcript messages. + + Args: + processor: The TranscriptProcessor that emitted the update + frame: TranscriptionUpdateFrame containing new messages + """ self.messages.extend(frame.messages) # Log the new messages logger.info("New transcript messages:") for msg in frame.messages: - logger.info(f"{msg.role}: {msg.content}") + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"{timestamp}{msg.role}: {msg.content}") - # Log the full transcript - logger.info("Full transcript:") - for msg in self.messages: - logger.info(f"{msg.role}: {msg.content}") + # # Log the full transcript + # logger.info("Full transcript:") + # for msg in self.messages: + # timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + # logger.info(f"{timestamp}{msg.role}: {msg.content}") async def main(): diff --git a/examples/foundational/28b-transcription-update-anthropic.py b/examples/foundational/28b-transcription-update-anthropic.py index 23ee93a21..1119efad2 100644 --- a/examples/foundational/28b-transcription-update-anthropic.py +++ b/examples/foundational/28b-transcription-update-anthropic.py @@ -32,7 +32,10 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptHandler: - """Simple handler to demonstrate transcript processing.""" + """Simple handler to demonstrate transcript processing. + + Maintains a list of conversation messages and logs them with timestamps. + """ def __init__(self): self.messages: List[TranscriptionMessage] = [] @@ -40,18 +43,25 @@ class TranscriptHandler: async def on_transcript_update( self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame ): - """Handle new transcript messages.""" + """Handle new transcript messages. + + Args: + processor: The TranscriptProcessor that emitted the update + frame: TranscriptionUpdateFrame containing new messages + """ self.messages.extend(frame.messages) # Log the new messages logger.info("New transcript messages:") for msg in frame.messages: - logger.info(f"{msg.role}: {msg.content}") + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"{timestamp}{msg.role}: {msg.content}") - # Log the full transcript - logger.info("Full transcript:") - for msg in self.messages: - logger.info(f"{msg.role}: {msg.content}") + # # Log the full transcript + # logger.info("Full transcript:") + # for msg in self.messages: + # timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + # logger.info(f"{timestamp}{msg.role}: {msg.content}") async def main(): diff --git a/examples/foundational/28c-transcription-update-gemini.py b/examples/foundational/28c-transcription-update-gemini.py index 27291a7c9..bf9448199 100644 --- a/examples/foundational/28c-transcription-update-gemini.py +++ b/examples/foundational/28c-transcription-update-gemini.py @@ -33,7 +33,10 @@ logger.add(sys.stderr, level="DEBUG") class TranscriptHandler: - """Simple handler to demonstrate transcript processing.""" + """Simple handler to demonstrate transcript processing. + + Maintains a list of conversation messages and logs them with timestamps. + """ def __init__(self): self.messages: List[TranscriptionMessage] = [] @@ -41,18 +44,25 @@ class TranscriptHandler: async def on_transcript_update( self, processor: TranscriptProcessor, frame: TranscriptionUpdateFrame ): - """Handle new transcript messages.""" + """Handle new transcript messages. + + Args: + processor: The TranscriptProcessor that emitted the update + frame: TranscriptionUpdateFrame containing new messages + """ self.messages.extend(frame.messages) # Log the new messages logger.info("New transcript messages:") for msg in frame.messages: - logger.info(f"{msg.role}: {msg.content}") + timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + logger.info(f"{timestamp}{msg.role}: {msg.content}") - # Log the full transcript - logger.info("Full transcript:") - for msg in self.messages: - logger.info(f"{msg.role}: {msg.content}") + # # Log the full transcript + # logger.info("Full transcript:") + # for msg in self.messages: + # timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" + # logger.info(f"{timestamp}{msg.role}: {msg.content}") async def main(): diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index f74d30371..ab8a6f6ad 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -207,6 +207,20 @@ class InterimTranscriptionFrame(TextFrame): return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})" +@dataclass +class OpenAILLMContextUserTimestampFrame(DataFrame): + """Timestamp information for user message in LLM context.""" + + timestamp: str + + +@dataclass +class OpenAILLMContextAssistantTimestampFrame(DataFrame): + """Timestamp information for assistant message in LLM context.""" + + timestamp: str + + @dataclass class TranscriptionMessage: """A message in a conversation transcript containing the role and content. diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 479746471..612375da2 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -15,6 +15,7 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMMessagesUpdateFrame, LLMSetToolsFrame, + OpenAILLMContextUserTimestampFrame, StartInterruptionFrame, TextFrame, TranscriptionFrame, @@ -26,6 +27,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContextFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.utils.time import time_now_iso8601 class LLMResponseAggregator(FrameProcessor): @@ -289,6 +291,10 @@ class LLMContextAggregator(LLMResponseAggregator): frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextUserTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + # Reset our accumulator state. self._reset() diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 97173f967..fcd4bfe52 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -4,13 +4,15 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import List +from typing import List, Optional from loguru import logger from pipecat.frames.frames import ( ErrorFrame, Frame, + OpenAILLMContextAssistantTimestampFrame, + OpenAILLMContextUserTimestampFrame, TranscriptionMessage, TranscriptionUpdateFrame, ) @@ -19,12 +21,12 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class TranscriptProcessor(FrameProcessor): - """Processes LLM context frames to generate conversation transcripts. + """Processes LLM context frames to generate timestamped conversation transcripts. - This processor monitors OpenAILLMContextFrame frames and extracts conversation - content, filtering out system messages and function calls. When new messages - are detected, it emits a TranscriptionUpdateFrame containing only the new - messages. + This processor monitors OpenAILLMContextFrame frames and their corresponding + timestamp frames to build a chronological conversation transcript. Messages are + stored by role until their matching timestamp frame arrives, then emitted via + TranscriptionUpdateFrame. Each LLM context (OpenAI, Anthropic, Google) provides conversion to the standard format: [ @@ -39,8 +41,8 @@ class TranscriptProcessor(FrameProcessor): ] Events: - on_transcript_update: Emitted when new transcript messages are available. - Args: TranscriptionUpdateFrame containing new messages. + on_transcript_update: Emitted when timestamped messages are available. + Args: TranscriptionUpdateFrame containing timestamped messages. Example: ```python @@ -49,7 +51,7 @@ class TranscriptProcessor(FrameProcessor): @transcript_processor.event_handler("on_transcript_update") async def on_transcript_update(processor, frame): for msg in frame.messages: - print(f"{msg.role}: {msg.content}") + print(f"[{msg.timestamp}] {msg.role}: {msg.content}") ``` """ @@ -62,6 +64,8 @@ class TranscriptProcessor(FrameProcessor): super().__init__(**kwargs) self._processed_messages: List[TranscriptionMessage] = [] self._register_event_handler("on_transcript_update") + self._pending_user_messages: List[TranscriptionMessage] = [] + self._pending_assistant_messages: List[TranscriptionMessage] = [] def _extract_messages(self, messages: List[dict]) -> List[TranscriptionMessage]: """Extract conversation messages from standard format. @@ -112,7 +116,16 @@ class TranscriptProcessor(FrameProcessor): return current[processed_len:] async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames, watching for OpenAILLMContextFrame. + """Process frames to build a timestamped conversation transcript. + + Handles three frame types in sequence: + 1. OpenAILLMContextFrame: Contains new messages to be timestamped + 2. OpenAILLMContextUserTimestampFrame: Timestamp for user messages + 3. OpenAILLMContextAssistantTimestampFrame: Timestamp for assistant messages + + Messages are stored by role until their corresponding timestamp frame arrives. + When a timestamp frame is received, the matching messages are timestamped and + emitted in chronological order via TranscriptionUpdateFrame. Args: frame: The frame to process @@ -124,27 +137,42 @@ class TranscriptProcessor(FrameProcessor): await super().process_frame(frame, direction) if isinstance(frame, OpenAILLMContextFrame): - try: - # Convert context messages to standard format - standard_messages = [] - for msg in frame.context.messages: - converted = frame.context.to_standard_messages(msg) - standard_messages.extend(converted) + # Extract and store messages by role + standard_messages = [] + for msg in frame.context.messages: + converted = frame.context.to_standard_messages(msg) + standard_messages.extend(converted) - # Extract and process messages - current_messages = self._extract_messages(standard_messages) - new_messages = self._find_new_messages(current_messages) + current_messages = self._extract_messages(standard_messages) + new_messages = self._find_new_messages(current_messages) - if new_messages: - # Update state and notify listeners - self._processed_messages.extend(new_messages) - update_frame = TranscriptionUpdateFrame(messages=new_messages) - await self._call_event_handler("on_transcript_update", update_frame) - await self.push_frame(update_frame) + # Store new messages by role + for msg in new_messages: + if msg.role == "user": + self._pending_user_messages.append(msg) + elif msg.role == "assistant": + self._pending_assistant_messages.append(msg) - except Exception as e: - logger.error(f"Error processing transcript in {self}: {e}") - await self.push_error(ErrorFrame(str(e))) + elif isinstance(frame, OpenAILLMContextUserTimestampFrame): + # Process pending user messages with timestamp + if self._pending_user_messages: + for msg in self._pending_user_messages: + msg.timestamp = frame.timestamp + self._processed_messages.extend(self._pending_user_messages) + update_frame = TranscriptionUpdateFrame(messages=self._pending_user_messages) + await self._call_event_handler("on_transcript_update", update_frame) + await self.push_frame(update_frame) + self._pending_user_messages = [] + + elif isinstance(frame, OpenAILLMContextAssistantTimestampFrame): + # Process pending assistant messages with timestamp + if self._pending_assistant_messages: + for msg in self._pending_assistant_messages: + msg.timestamp = frame.timestamp + self._processed_messages.extend(self._pending_assistant_messages) + update_frame = TranscriptionUpdateFrame(messages=self._pending_assistant_messages) + await self._call_event_handler("on_transcript_update", update_frame) + await self.push_frame(update_frame) + self._pending_assistant_messages = [] - # Always push the original frame downstream await self.push_frame(frame, direction) diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index d8f485296..93cff6f9e 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -26,6 +26,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMUpdateSettingsFrame, + OpenAILLMContextAssistantTimestampFrame, StartInterruptionFrame, TextFrame, UserImageRawFrame, @@ -43,6 +44,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import LLMService +from pipecat.utils.time import time_now_iso8601 try: from anthropic import NOT_GIVEN, AsyncAnthropic, NotGiven @@ -791,8 +793,13 @@ class AnthropicAssistantContextAggregator(LLMAssistantContextAggregator): if run_llm: await self._user_context_aggregator.push_context_frame() + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + except Exception as e: logger.error(f"Error processing frame: {e}") diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 383dde624..c7d32eff3 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -23,6 +23,8 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMUpdateSettingsFrame, + OpenAILLMContextAssistantTimestampFrame, + OpenAILLMContextUserTimestampFrame, TextFrame, TTSAudioRawFrame, TTSStartedFrame, @@ -41,6 +43,7 @@ from pipecat.services.openai import ( OpenAIUserContextAggregator, ) from pipecat.transcriptions.language import Language +from pipecat.utils.time import time_now_iso8601 try: import google.ai.generativelanguage as glm @@ -227,9 +230,14 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator): # if the tasks gets cancelled we won't be able to clear things up. self._aggregation = "" + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextUserTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + # Reset our accumulator state. self._reset() @@ -300,9 +308,14 @@ class GoogleAssistantContextAggregator(OpenAIAssistantContextAggregator): if run_llm: await self._user_context_aggregator.push_context_frame() + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + except Exception as e: logger.exception(f"Error processing frame: {e}") diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 85e1a95f0..159db8d2f 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -25,6 +25,7 @@ from pipecat.frames.frames import ( LLMFullResponseStartFrame, LLMMessagesFrame, LLMUpdateSettingsFrame, + OpenAILLMContextAssistantTimestampFrame, StartInterruptionFrame, TextFrame, TTSAudioRawFrame, @@ -46,6 +47,7 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_services import ImageGenService, LLMService, TTSService +from pipecat.utils.time import time_now_iso8601 try: from openai import ( @@ -597,8 +599,13 @@ class OpenAIAssistantContextAggregator(LLMAssistantContextAggregator): if run_llm: await self._user_context_aggregator.push_context_frame() + # Push context frame frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) + # Push timestamp frame with current time + timestamp_frame = OpenAILLMContextAssistantTimestampFrame(timestamp=time_now_iso8601()) + await self.push_frame(timestamp_frame) + except Exception as e: logger.error(f"Error processing frame: {e}") From b5bd662fe16b7daff28d7e376714095098142beb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 14 Dec 2024 12:08:38 -0500 Subject: [PATCH 52/90] Add changelog and rename examples --- CHANGELOG.md | 17 ++++++++++++++++- ...py => 28a-transcription-processor-openai.py} | 0 ...py => 28b-transcript-processor-anthropic.py} | 0 ...py => 28c-transcription-processor-gemini.py} | 0 4 files changed, 16 insertions(+), 1 deletion(-) rename examples/foundational/{28a-transcription-update-openai.py => 28a-transcription-processor-openai.py} (100%) rename examples/foundational/{28b-transcription-update-anthropic.py => 28b-transcript-processor-anthropic.py} (100%) rename examples/foundational/{28c-transcription-update-gemini.py => 28c-transcription-processor-gemini.py} (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e069fc7..5ecba8686 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `audioop-lts` (https://github.com/AbstractUmbra/audioop) to provide the same functionality. -- Added support for more languages to ElevenLabs (Arabic, Croatian, Filipino, +- Added timestamped conversation transcript support: + + - New `TranscriptProcessor` factory provides access to user and assistant + transcript processors. + - `UserTranscriptProcessor` processes user speech with timestamps from + transcription. + - `AssistantTranscriptProcessor` processes assistant responses with LLM + context timestamps. + - Messages emitted with ISO 8601 timestamps indicating when they were spoken. + - Supports all LLM formats (OpenAI, Anthropic, Google) via standard message + format. + - New examples: `28a-transcription-processor-openai.py`, + `28b-transcription-processor-anthropic.py`, and + `28c-transcription-processor-gemini.py` + +- Add support for more languages to ElevenLabs (Arabic, Croatian, Filipino, Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, Galician, Hebrew, Mandarin, Serbian, Tagalog, Urdu, Xhosa). diff --git a/examples/foundational/28a-transcription-update-openai.py b/examples/foundational/28a-transcription-processor-openai.py similarity index 100% rename from examples/foundational/28a-transcription-update-openai.py rename to examples/foundational/28a-transcription-processor-openai.py diff --git a/examples/foundational/28b-transcription-update-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py similarity index 100% rename from examples/foundational/28b-transcription-update-anthropic.py rename to examples/foundational/28b-transcript-processor-anthropic.py diff --git a/examples/foundational/28c-transcription-update-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py similarity index 100% rename from examples/foundational/28c-transcription-update-gemini.py rename to examples/foundational/28c-transcription-processor-gemini.py From 1f8a217cd1ef3da357cda2d968323d6648ca6560 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 16 Dec 2024 10:17:33 -0500 Subject: [PATCH 53/90] Code review changes --- CHANGELOG.md | 2 +- .../07a-interruptible-anthropic.py | 42 ++------------ .../28a-transcription-processor-openai.py | 2 +- .../28b-transcript-processor-anthropic.py | 2 +- src/pipecat/frames/frames.py | 30 +++++++++- .../aggregators/openai_llm_context.py | 58 ++++++------------- 6 files changed, 57 insertions(+), 79 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ecba8686..5e784cdba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 format. - New examples: `28a-transcription-processor-openai.py`, `28b-transcription-processor-anthropic.py`, and - `28c-transcription-processor-gemini.py` + `28c-transcription-processor-gemini.py`. - Add support for more languages to ElevenLabs (Arabic, Croatian, Filipino, Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index 25a301269..e7e680eab 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -7,7 +7,6 @@ import asyncio import os import sys -from typing import List import aiohttp from dotenv import load_dotenv @@ -15,13 +14,12 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import Frame, LLMMessagesFrame +from pipecat.frames.frames import LLMMessagesFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.anthropic import AnthropicLLMContext, AnthropicLLMService +from pipecat.services.anthropic import AnthropicLLMService from pipecat.services.cartesia import CartesiaTTSService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -31,28 +29,6 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -class TestAnthropicLLMService(AnthropicLLMService): - async def process_frame(self, frame: Frame, direction: FrameDirection): - if isinstance(frame, LLMMessagesFrame): - logger.info("Original OpenAI format messages:") - logger.info(frame.messages) - - # Convert to Anthropic format - context = AnthropicLLMContext.from_messages(frame.messages) - logger.info("Converted to Anthropic format:") - logger.info(context.messages) - - # Convert back to OpenAI format - openai_messages = [] - for msg in context.messages: - converted = context.to_standard_messages(msg) - openai_messages.extend(converted) - logger.info("Converted back to OpenAI format:") - logger.info(openai_messages) - - await super().process_frame(frame, direction) - - async def main(): async with aiohttp.ClientSession() as session: (room_url, token) = await configure(session) @@ -74,24 +50,18 @@ async def main(): voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) - llm = TestAnthropicLLMService( + llm = AnthropicLLMService( api_key=os.getenv("ANTHROPIC_API_KEY"), model="claude-3-opus-20240229" ) - # Test messages including various formats + # todo: think more about how to handle system prompts in a more general way. OpenAI, + # Google, and Anthropic all have slightly different approaches to providing a system + # prompt. messages = [ { "role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative, helpful, and brief way. Say hello.", }, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Hello! How can I help you today?"}, - {"type": "text", "text": "I'm ready to assist."}, - ], - }, - {"role": "user", "content": "Hi there!"}, ] context = OpenAILLMContext(messages) diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index 390343465..1e8463b69 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -127,7 +127,7 @@ async def main(): async def on_first_participant_joined(transport, participant): await transport.capture_participant_transcription(participant["id"]) # Kick off the conversation. - await task.queue_frames([LLMMessagesFrame(messages)]) + await task.queue_frames([context_aggregator.user().get_context_frame()]) runner = PipelineRunner() diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index 1119efad2..626206c5f 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -127,7 +127,7 @@ async def main(): async def on_first_participant_joined(transport, participant): await transport.capture_participant_transcription(participant["id"]) # Kick off the conversation. - await task.queue_frames([LLMMessagesFrame(messages)]) + await task.queue_frames([context_aggregator.user().get_context_frame()]) runner = PipelineRunner() diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index ab8a6f6ad..d02112a6f 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -5,7 +5,7 @@ # from dataclasses import dataclass, field -from typing import Any, List, Literal, Mapping, Optional, Tuple, TypeAlias +from typing import Any, List, Literal, Mapping, Optional, Tuple from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.clocks.base_clock import BaseClock @@ -240,6 +240,34 @@ class TranscriptionUpdateFrame(DataFrame): This frame is emitted when new messages are added to the conversation history, containing only the newly added messages rather than the full transcript. Messages have normalized roles (user/assistant) regardless of the LLM service used. + Messages are always in the OpenAI standard message format, which supports both: + + Simple format: + [ + { + "role": "user", + "content": "Hi, how are you?" + }, + { + "role": "assistant", + "content": "Great! And you?" + } + ] + + Content list format: + [ + { + "role": "user", + "content": [{"type": "text", "text": "Hi, how are you?"}] + }, + { + "role": "assistant", + "content": [{"type": "text", "text": "Great! And you?"}] + } + ] + + OpenAI supports both formats. Anthropic and Google messages are converted to the + content list format. """ messages: List[TranscriptionMessage] diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 4adf76de0..853ac1baa 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -112,59 +112,39 @@ class OpenAILLMContext: msgs.append(msg) return json.dumps(msgs) - def from_standard_message(self, message) -> dict: - """Convert standard format message to OpenAI format. + def from_standard_message(self, message): + """Convert from OpenAI message format to OpenAI message format (passthrough). - Converts structured content back to OpenAI's simple string format. + OpenAI's format allows both simple string content and structured content: + - Simple: {"role": "user", "content": "Hello"} + - Structured: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} + + Since OpenAI is our standard format, this is a passthrough function. Args: - message: Message in standard format: - { - "role": "user/assistant", - "content": [{"type": "text", "text": str}] - } + message (dict): Message in OpenAI format Returns: - Message in OpenAI format: - { - "role": "user/assistant", - "content": str - } + dict: Same message, unchanged """ - # If content is already a string, return as-is - if isinstance(message.get("content"), str): - return message - - # Convert structured content to string - if isinstance(message.get("content"), list): - text_parts = [] - for part in message["content"]: - if part.get("type") == "text": - text_parts.append(part["text"]) - - return {"role": message["role"], "content": " ".join(text_parts) if text_parts else ""} - return message def to_standard_messages(self, obj) -> list: - """Convert OpenAI message to standard structured format. + """Convert from OpenAI message format to OpenAI message format (passthrough). + + OpenAI's format is our standard format throughout Pipecat. This function + returns a list containing the original message to maintain consistency with + other LLM services that may need to return multiple messages. Args: - obj: Message in OpenAI format {"role": "user", "content": "text"} + obj (dict): Message in OpenAI format with either: + - Simple content: {"role": "user", "content": "Hello"} + - List content: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} Returns: - List containing message with structured content: - [{"role": "user", "content": [{"type": "text", "text": "message"}]}] + list: List containing the original messages, preserving whether + the content was in simple string or structured list format """ - # Skip messages without content - if not obj.get("content"): - return [] - - # Convert simple string content to structured format - if isinstance(obj["content"], str): - return [{"role": obj["role"], "content": [{"type": "text", "text": obj["content"]}]}] - - # Return original message if content is already structured return [obj] def get_messages_for_initializing_history(self): From 4211664a77fb605f3d3a2bca2c86f3ec3b26aab0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 16 Dec 2024 10:33:34 -0500 Subject: [PATCH 54/90] TranscriptProcessor to handle simple and list content --- src/pipecat/processors/transcript_processor.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index fcd4bfe52..be53cd79a 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -71,7 +71,9 @@ class TranscriptProcessor(FrameProcessor): """Extract conversation messages from standard format. Args: - messages: List of messages in standard format with structured content + messages: List of messages in OpenAI format, which can be either: + - Simple format: {"role": "user", "content": "Hello"} + - Content list: {"role": "user", "content": [{"type": "text", "text": "Hello"}]} Returns: List[TranscriptionMessage]: Normalized conversation messages @@ -82,9 +84,17 @@ class TranscriptProcessor(FrameProcessor): if msg["role"] not in ("user", "assistant"): continue - content = msg.get("content", []) - if isinstance(content, list): - # Extract text from structured content + if "content" not in msg: + logger.warning(f"Message missing content field: {msg}") + continue + + content = msg.get("content") + if isinstance(content, str): + # Handle simple string content + if content: + result.append(TranscriptionMessage(role=msg["role"], content=content)) + elif isinstance(content, list): + # Handle structured content text_parts = [] for part in content: if isinstance(part, dict) and part.get("type") == "text": From 1117c2148365caca2202bef57792438ba8cfb27e Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 16 Dec 2024 15:24:58 -0500 Subject: [PATCH 55/90] Refactor TranscriptProcessor into user and assistant processors --- CHANGELOG.md | 3 +- .../28a-transcription-processor-openai.py | 23 +- .../28b-transcript-processor-anthropic.py | 31 ++- .../28c-transcription-processor-gemini.py | 41 ++-- src/pipecat/frames/frames.py | 7 - .../processors/aggregators/llm_response.py | 6 - .../processors/transcript_processor.py | 227 +++++++++++------- src/pipecat/services/google.py | 5 - 8 files changed, 185 insertions(+), 158 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e784cdba..e3f2c2dc7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,9 +25,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Messages emitted with ISO 8601 timestamps indicating when they were spoken. - Supports all LLM formats (OpenAI, Anthropic, Google) via standard message format. + - Shared event handling for both user and assistant transcript updates. - New examples: `28a-transcription-processor-openai.py`, `28b-transcription-processor-anthropic.py`, and - `28c-transcription-processor-gemini.py`. + `28c-transcription-processor-gemini.py` - Add support for more languages to ElevenLabs (Arabic, Croatian, Filipino, Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index 1e8463b69..0966c882f 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -15,13 +15,14 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.openai import OpenAILLMService from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -57,12 +58,6 @@ class TranscriptHandler: timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" logger.info(f"{timestamp}{msg.role}: {msg.content}") - # # Log the full transcript - # logger.info("Full transcript:") - # for msg in self.messages: - # timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" - # logger.info(f"{timestamp}{msg.role}: {msg.content}") - async def main(): async with aiohttp.ClientSession() as session: @@ -70,16 +65,18 @@ async def main(): transport = DailyTransport( room_url, - token, + None, "Respond bot", DailyParams( audio_out_enabled=True, - transcription_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, ), ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady @@ -101,23 +98,25 @@ async def main(): context_aggregator = llm.create_context_aggregator(context) # Create transcript processor and handler - transcript_processor = TranscriptProcessor() + transcript = TranscriptProcessor() transcript_handler = TranscriptHandler() # Register event handler for transcript updates - @transcript_processor.event_handler("on_transcript_update") + @transcript.event_handler("on_transcript_update") async def on_transcript_update(processor, frame): await transcript_handler.on_transcript_update(processor, frame) pipeline = Pipeline( [ transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM tts, # TTS transport.output(), # Transport bot output context_aggregator.assistant(), # Assistant spoken responses - transcript_processor, # Process transcripts + transcript.assistant(), # Assistant transcripts ] ) diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index 626206c5f..066828652 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -15,7 +15,7 @@ from loguru import logger from runner import configure from pipecat.audio.vad.silero import SileroVADAnalyzer -from pipecat.frames.frames import LLMMessagesFrame, TranscriptionMessage, TranscriptionUpdateFrame +from pipecat.frames.frames import TranscriptionMessage, TranscriptionUpdateFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -23,6 +23,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.anthropic import AnthropicLLMService from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService from pipecat.transports.services.daily import DailyParams, DailyTransport load_dotenv(override=True) @@ -57,12 +58,6 @@ class TranscriptHandler: timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" logger.info(f"{timestamp}{msg.role}: {msg.content}") - # # Log the full transcript - # logger.info("Full transcript:") - # for msg in self.messages: - # timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" - # logger.info(f"{timestamp}{msg.role}: {msg.content}") - async def main(): async with aiohttp.ClientSession() as session: @@ -70,16 +65,18 @@ async def main(): transport = DailyTransport( room_url, - token, + None, "Respond bot", DailyParams( audio_out_enabled=True, - transcription_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, ), ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady @@ -101,23 +98,20 @@ async def main(): context_aggregator = llm.create_context_aggregator(context) # Create transcript processor and handler - transcript_processor = TranscriptProcessor() + transcript = TranscriptProcessor() transcript_handler = TranscriptHandler() - # Register event handler for transcript updates - @transcript_processor.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - await transcript_handler.on_transcript_update(processor, frame) - pipeline = Pipeline( [ transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts context_aggregator.user(), # User responses llm, # LLM tts, # TTS transport.output(), # Transport bot output context_aggregator.assistant(), # Assistant spoken responses - transcript_processor, # Process transcripts + transcript.assistant(), # Assistant transcripts ] ) @@ -129,6 +123,11 @@ async def main(): # Kick off the conversation. await task.queue_frames([context_aggregator.user().get_context_frame()]) + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + runner = PipelineRunner() await runner.run(task) diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py index bf9448199..6c8118c57 100644 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -22,6 +22,7 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.deepgram import DeepgramSTTService from pipecat.services.google import GoogleLLMService from pipecat.services.openai import OpenAILLMContext from pipecat.transports.services.daily import DailyParams, DailyTransport @@ -58,12 +59,6 @@ class TranscriptHandler: timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" logger.info(f"{timestamp}{msg.role}: {msg.content}") - # # Log the full transcript - # logger.info("Full transcript:") - # for msg in self.messages: - # timestamp = f"[{msg.timestamp}] " if msg.timestamp else "" - # logger.info(f"{timestamp}{msg.role}: {msg.content}") - async def main(): async with aiohttp.ClientSession() as session: @@ -71,16 +66,18 @@ async def main(): transport = DailyTransport( room_url, - token, + None, "Respond bot", DailyParams( audio_out_enabled=True, - transcription_enabled=True, vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), + vad_audio_passthrough=True, ), ) + stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) + tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady @@ -104,23 +101,20 @@ async def main(): context_aggregator = llm.create_context_aggregator(context) # Create transcript processor and handler - transcript_processor = TranscriptProcessor() + transcript = TranscriptProcessor() transcript_handler = TranscriptHandler() - # Register event handler for transcript updates - @transcript_processor.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - await transcript_handler.on_transcript_update(processor, frame) - pipeline = Pipeline( [ - transport.input(), - context_aggregator.user(), - llm, - tts, - transport.output(), - context_aggregator.assistant(), - transcript_processor, + transport.input(), # Transport user input + stt, # STT + transcript.user(), # User transcripts + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + transcript.assistant(), # Assistant transcripts ] ) @@ -139,6 +133,11 @@ async def main(): # Kick off the conversation. await task.queue_frames([context_aggregator.user().get_context_frame()]) + # Register event handler for transcript updates + @transcript.event_handler("on_transcript_update") + async def on_transcript_update(processor, frame): + await transcript_handler.on_transcript_update(processor, frame) + runner = PipelineRunner() await runner.run(task) diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index d02112a6f..e9d942b4e 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -207,13 +207,6 @@ class InterimTranscriptionFrame(TextFrame): return f"{self.name}(user: {self.user_id}, text: [{self.text}], language: {self.language}, timestamp: {self.timestamp})" -@dataclass -class OpenAILLMContextUserTimestampFrame(DataFrame): - """Timestamp information for user message in LLM context.""" - - timestamp: str - - @dataclass class OpenAILLMContextAssistantTimestampFrame(DataFrame): """Timestamp information for assistant message in LLM context.""" diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 612375da2..479746471 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -15,7 +15,6 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMMessagesUpdateFrame, LLMSetToolsFrame, - OpenAILLMContextUserTimestampFrame, StartInterruptionFrame, TextFrame, TranscriptionFrame, @@ -27,7 +26,6 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContextFrame, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -from pipecat.utils.time import time_now_iso8601 class LLMResponseAggregator(FrameProcessor): @@ -291,10 +289,6 @@ class LLMContextAggregator(LLMResponseAggregator): frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextUserTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - # Reset our accumulator state. self._reset() diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index be53cd79a..a95e502a8 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -4,7 +4,8 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from typing import List, Optional +from abc import ABC, abstractmethod +from typing import List from loguru import logger @@ -12,7 +13,7 @@ from pipecat.frames.frames import ( ErrorFrame, Frame, OpenAILLMContextAssistantTimestampFrame, - OpenAILLMContextUserTimestampFrame, + TranscriptionFrame, TranscriptionMessage, TranscriptionUpdateFrame, ) @@ -20,55 +21,72 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFr from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -class TranscriptProcessor(FrameProcessor): - """Processes LLM context frames to generate timestamped conversation transcripts. +class BaseTranscriptProcessor(FrameProcessor, ABC): + """Base class for processing conversation transcripts. - This processor monitors OpenAILLMContextFrame frames and their corresponding - timestamp frames to build a chronological conversation transcript. Messages are - stored by role until their matching timestamp frame arrives, then emitted via - TranscriptionUpdateFrame. - - Each LLM context (OpenAI, Anthropic, Google) provides conversion to the standard format: - [ - { - "role": "user", - "content": [{"type": "text", "text": "Hi, how are you?"}] - }, - { - "role": "assistant", - "content": [{"type": "text", "text": "Great! And you?"}] - } - ] - - Events: - on_transcript_update: Emitted when timestamped messages are available. - Args: TranscriptionUpdateFrame containing timestamped messages. - - Example: - ```python - transcript_processor = TranscriptProcessor() - - @transcript_processor.event_handler("on_transcript_update") - async def on_transcript_update(processor, frame): - for msg in frame.messages: - print(f"[{msg.timestamp}] {msg.role}: {msg.content}") - ``` + Provides common functionality for handling transcript messages and updates. """ def __init__(self, **kwargs): - """Initialize the transcript processor. - - Args: - **kwargs: Additional arguments passed to FrameProcessor - """ + """Initialize processor with empty message store.""" super().__init__(**kwargs) self._processed_messages: List[TranscriptionMessage] = [] self._register_event_handler("on_transcript_update") - self._pending_user_messages: List[TranscriptionMessage] = [] + + async def _emit_update(self, messages: List[TranscriptionMessage]): + """Emit transcript updates for new messages. + + Args: + messages: New messages to emit in update + """ + if messages: + self._processed_messages.extend(messages) + update_frame = TranscriptionUpdateFrame(messages=messages) + await self._call_event_handler("on_transcript_update", update_frame) + await self.push_frame(update_frame) + + @abstractmethod + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process incoming frames to build conversation transcript. + + Args: + frame: Input frame to process + direction: Frame processing direction + """ + await super().process_frame(frame, direction) + + +class UserTranscriptProcessor(BaseTranscriptProcessor): + """Processes user transcription frames into timestamped conversation messages.""" + + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process TranscriptionFrames into user conversation messages. + + Args: + frame: Input frame to process + direction: Frame processing direction + """ + await super().process_frame(frame, direction) + + if isinstance(frame, TranscriptionFrame): + message = TranscriptionMessage( + role="user", content=frame.text, timestamp=frame.timestamp + ) + await self._emit_update([message]) + + await self.push_frame(frame, direction) + + +class AssistantTranscriptProcessor(BaseTranscriptProcessor): + """Processes assistant LLM context frames into timestamped conversation messages.""" + + def __init__(self, **kwargs): + """Initialize processor with empty message stores.""" + super().__init__(**kwargs) self._pending_assistant_messages: List[TranscriptionMessage] = [] def _extract_messages(self, messages: List[dict]) -> List[TranscriptionMessage]: - """Extract conversation messages from standard format. + """Extract assistant messages from the OpenAI standard message format. Args: messages: List of messages in OpenAI format, which can be either: @@ -80,21 +98,14 @@ class TranscriptProcessor(FrameProcessor): """ result = [] for msg in messages: - # Only process user and assistant messages - if msg["role"] not in ("user", "assistant"): - continue - - if "content" not in msg: - logger.warning(f"Message missing content field: {msg}") + if msg["role"] != "assistant": continue content = msg.get("content") if isinstance(content, str): - # Handle simple string content if content: - result.append(TranscriptionMessage(role=msg["role"], content=content)) + result.append(TranscriptionMessage(role="assistant", content=content)) elif isinstance(content, list): - # Handle structured content text_parts = [] for part in content: if isinstance(part, dict) and part.get("type") == "text": @@ -102,13 +113,13 @@ class TranscriptProcessor(FrameProcessor): if text_parts: result.append( - TranscriptionMessage(role=msg["role"], content=" ".join(text_parts)) + TranscriptionMessage(role="assistant", content=" ".join(text_parts)) ) return result def _find_new_messages(self, current: List[TranscriptionMessage]) -> List[TranscriptionMessage]: - """Find messages in current that aren't in self._processed_messages. + """Find unprocessed messages from current list. Args: current: List of current messages @@ -126,28 +137,15 @@ class TranscriptProcessor(FrameProcessor): return current[processed_len:] async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process frames to build a timestamped conversation transcript. - - Handles three frame types in sequence: - 1. OpenAILLMContextFrame: Contains new messages to be timestamped - 2. OpenAILLMContextUserTimestampFrame: Timestamp for user messages - 3. OpenAILLMContextAssistantTimestampFrame: Timestamp for assistant messages - - Messages are stored by role until their corresponding timestamp frame arrives. - When a timestamp frame is received, the matching messages are timestamped and - emitted in chronological order via TranscriptionUpdateFrame. + """Process frames into assistant conversation messages. Args: - frame: The frame to process + frame: Input frame to process direction: Frame processing direction - - Raises: - ErrorFrame: If message processing fails """ await super().process_frame(frame, direction) if isinstance(frame, OpenAILLMContextFrame): - # Extract and store messages by role standard_messages = [] for msg in frame.context.messages: converted = frame.context.to_standard_messages(msg) @@ -155,34 +153,83 @@ class TranscriptProcessor(FrameProcessor): current_messages = self._extract_messages(standard_messages) new_messages = self._find_new_messages(current_messages) - - # Store new messages by role - for msg in new_messages: - if msg.role == "user": - self._pending_user_messages.append(msg) - elif msg.role == "assistant": - self._pending_assistant_messages.append(msg) - - elif isinstance(frame, OpenAILLMContextUserTimestampFrame): - # Process pending user messages with timestamp - if self._pending_user_messages: - for msg in self._pending_user_messages: - msg.timestamp = frame.timestamp - self._processed_messages.extend(self._pending_user_messages) - update_frame = TranscriptionUpdateFrame(messages=self._pending_user_messages) - await self._call_event_handler("on_transcript_update", update_frame) - await self.push_frame(update_frame) - self._pending_user_messages = [] + self._pending_assistant_messages.extend(new_messages) elif isinstance(frame, OpenAILLMContextAssistantTimestampFrame): - # Process pending assistant messages with timestamp if self._pending_assistant_messages: for msg in self._pending_assistant_messages: msg.timestamp = frame.timestamp - self._processed_messages.extend(self._pending_assistant_messages) - update_frame = TranscriptionUpdateFrame(messages=self._pending_assistant_messages) - await self._call_event_handler("on_transcript_update", update_frame) - await self.push_frame(update_frame) + await self._emit_update(self._pending_assistant_messages) self._pending_assistant_messages = [] await self.push_frame(frame, direction) + + +class TranscriptProcessor: + """Factory for creating and managing transcript processors. + + Provides unified access to user and assistant transcript processors + with shared event handling. + + Example: + ```python + transcript = TranscriptProcessor() + + pipeline = Pipeline( + [ + transport.input(), + stt, + transcript.user(), # User transcripts + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + transcript.assistant(), # Assistant transcripts + ] + ) + + @transcript.event_handler("on_transcript_update") + async def handle_update(processor, frame): + print(f"New messages: {frame.messages}") + ``` + """ + + def __init__(self, **kwargs): + """Initialize factory with user and assistant processors.""" + self._user_processor = UserTranscriptProcessor(**kwargs) + self._assistant_processor = AssistantTranscriptProcessor(**kwargs) + self._event_handlers = {} + + def user(self) -> UserTranscriptProcessor: + """Get the user transcript processor.""" + return self._user_processor + + def assistant(self) -> AssistantTranscriptProcessor: + """Get the assistant transcript processor.""" + return self._assistant_processor + + def event_handler(self, event_name: str): + """Register event handler for both processors. + + Args: + event_name: Name of event to handle + + Returns: + Decorator function that registers handler with both processors + """ + + def decorator(handler): + self._event_handlers[event_name] = handler + + @self._user_processor.event_handler(event_name) + async def user_handler(processor, frame): + return await handler(processor, frame) + + @self._assistant_processor.event_handler(event_name) + async def assistant_handler(processor, frame): + return await handler(processor, frame) + + return handler + + return decorator diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index c7d32eff3..6bbf1d000 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -24,7 +24,6 @@ from pipecat.frames.frames import ( LLMMessagesFrame, LLMUpdateSettingsFrame, OpenAILLMContextAssistantTimestampFrame, - OpenAILLMContextUserTimestampFrame, TextFrame, TTSAudioRawFrame, TTSStartedFrame, @@ -234,10 +233,6 @@ class GoogleUserContextAggregator(OpenAIUserContextAggregator): frame = OpenAILLMContextFrame(self._context) await self.push_frame(frame) - # Push timestamp frame with current time - timestamp_frame = OpenAILLMContextUserTimestampFrame(timestamp=time_now_iso8601()) - await self.push_frame(timestamp_frame) - # Reset our accumulator state. self._reset() From 8a7a61914edbd80083d4215006b29f6c22cd4c39 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 17 Dec 2024 22:30:23 -0500 Subject: [PATCH 56/90] Code review feedback --- CHANGELOG.md | 3 +- .../processors/transcript_processor.py | 75 ++++++++++++------- 2 files changed, 47 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e3f2c2dc7..5e784cdba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,10 +25,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Messages emitted with ISO 8601 timestamps indicating when they were spoken. - Supports all LLM formats (OpenAI, Anthropic, Google) via standard message format. - - Shared event handling for both user and assistant transcript updates. - New examples: `28a-transcription-processor-openai.py`, `28b-transcription-processor-anthropic.py`, and - `28c-transcription-processor-gemini.py` + `28c-transcription-processor-gemini.py`. - Add support for more languages to ElevenLabs (Arabic, Croatian, Filipino, Tamil) and PlayHT (Afrikans, Albanian, Amharic, Arabic, Bengali, Croatian, diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index a95e502a8..1e5e97c59 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -4,13 +4,9 @@ # SPDX-License-Identifier: BSD 2-Clause License # -from abc import ABC, abstractmethod from typing import List -from loguru import logger - from pipecat.frames.frames import ( - ErrorFrame, Frame, OpenAILLMContextAssistantTimestampFrame, TranscriptionFrame, @@ -21,7 +17,7 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContextFr from pipecat.processors.frame_processor import FrameDirection, FrameProcessor -class BaseTranscriptProcessor(FrameProcessor, ABC): +class BaseTranscriptProcessor(FrameProcessor): """Base class for processing conversation transcripts. Provides common functionality for handling transcript messages and updates. @@ -45,16 +41,6 @@ class BaseTranscriptProcessor(FrameProcessor, ABC): await self._call_event_handler("on_transcript_update", update_frame) await self.push_frame(update_frame) - @abstractmethod - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process incoming frames to build conversation transcript. - - Args: - frame: Input frame to process - direction: Frame processing direction - """ - await super().process_frame(frame, direction) - class UserTranscriptProcessor(BaseTranscriptProcessor): """Processes user transcription frames into timestamped conversation messages.""" @@ -195,18 +181,44 @@ class TranscriptProcessor: ``` """ - def __init__(self, **kwargs): - """Initialize factory with user and assistant processors.""" - self._user_processor = UserTranscriptProcessor(**kwargs) - self._assistant_processor = AssistantTranscriptProcessor(**kwargs) + def __init__(self): + """Initialize factory.""" + self._user_processor = None + self._assistant_processor = None self._event_handlers = {} - def user(self) -> UserTranscriptProcessor: - """Get the user transcript processor.""" + def user(self, **kwargs) -> UserTranscriptProcessor: + """Get the user transcript processor. + + Args: + **kwargs: Arguments specific to UserTranscriptProcessor + """ + if self._user_processor is None: + self._user_processor = UserTranscriptProcessor(**kwargs) + # Apply any registered event handlers + for event_name, handler in self._event_handlers.items(): + + @self._user_processor.event_handler(event_name) + async def user_handler(processor, frame): + return await handler(processor, frame) + return self._user_processor - def assistant(self) -> AssistantTranscriptProcessor: - """Get the assistant transcript processor.""" + def assistant(self, **kwargs) -> AssistantTranscriptProcessor: + """Get the assistant transcript processor. + + Args: + **kwargs: Arguments specific to AssistantTranscriptProcessor + """ + if self._assistant_processor is None: + self._assistant_processor = AssistantTranscriptProcessor(**kwargs) + # Apply any registered event handlers + for event_name, handler in self._event_handlers.items(): + + @self._assistant_processor.event_handler(event_name) + async def assistant_handler(processor, frame): + return await handler(processor, frame) + return self._assistant_processor def event_handler(self, event_name: str): @@ -222,13 +234,18 @@ class TranscriptProcessor: def decorator(handler): self._event_handlers[event_name] = handler - @self._user_processor.event_handler(event_name) - async def user_handler(processor, frame): - return await handler(processor, frame) + # Apply handler to existing processors if they exist + if self._user_processor: - @self._assistant_processor.event_handler(event_name) - async def assistant_handler(processor, frame): - return await handler(processor, frame) + @self._user_processor.event_handler(event_name) + async def user_handler(processor, frame): + return await handler(processor, frame) + + if self._assistant_processor: + + @self._assistant_processor.event_handler(event_name) + async def assistant_handler(processor, frame): + return await handler(processor, frame) return handler From 3b3e22fe7c1ed2a2277ee0e88c80d94e8b8d7373 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 18 Dec 2024 18:11:31 -0500 Subject: [PATCH 57/90] Add model parameter to OpenAI realtime service constructor, update default model --- CHANGELOG.md | 5 +++++ src/pipecat/services/openai_realtime_beta/openai.py | 8 +++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78e069fc7..7b532fbd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `OpenAIRealtimeBetaLLMService` now takes a `model` parameter in the + constructor. + +- Updated the default model for the `OpenAIRealtimeBetaLLMService`. + - Room expiration (`exp`) in `DailyRoomProperties` is now optional (`None`) by default instead of automatically setting a 5-minute expiration time. You must explicitly set expiration time if desired. diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index c0240676e..2e952bea1 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -72,15 +72,17 @@ class OpenAIRealtimeBetaLLMService(LLMService): self, *, api_key: str, - base_url="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01", + model: str = "gpt-4o-realtime-preview-2024-12-17", + base_url: str = "wss://api.openai.com/v1/realtime", session_properties: events.SessionProperties = events.SessionProperties(), start_audio_paused: bool = False, send_transcription_frames: bool = True, **kwargs, ): - super().__init__(base_url=base_url, **kwargs) + full_url = f"{base_url}?model={model}" + super().__init__(base_url=full_url, **kwargs) self.api_key = api_key - self.base_url = base_url + self.base_url = full_url self._session_properties: events.SessionProperties = session_properties self._audio_input_paused = start_audio_paused From f89014d10028cda6e7f10b31c408b0eab0ac4463 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 16:01:25 -0800 Subject: [PATCH 58/90] pyproject: update langchaing to 0.3.12 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4cf1de72c..be9eab02f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ gstreamer = [ "pygobject~=3.48.2" ] fireworks = [ "openai~=1.57.2" ] krisp = [ "pipecat-ai-krisp~=0.3.0" ] koala = [ "pvkoala~=2.0.2" ] -langchain = [ "langchain~=0.2.14", "langchain-community~=0.2.12", "langchain-openai~=0.1.20" ] +langchain = [ "langchain~=0.3.12", "langchain-community~=0.3.12", "langchain-openai~=0.2.12" ] livekit = [ "livekit~=0.17.5", "livekit-api~=0.7.1" ] lmnt = [ "lmnt~=1.1.4" ] local = [ "pyaudio~=0.2.14" ] From 4f093f11dbe537d14c217a04eefe4a3618841873 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 18 Dec 2024 11:38:57 -0500 Subject: [PATCH 59/90] Add CerebrasLLMService and foundational example --- CHANGELOG.md | 3 + README.md | 25 +-- .../14k-function-calling-cerebras.py | 148 ++++++++++++++++++ pyproject.toml | 1 + src/pipecat/services/cerebras.py | 38 +++++ 5 files changed, 203 insertions(+), 12 deletions(-) create mode 100644 examples/foundational/14k-function-calling-cerebras.py create mode 100644 src/pipecat/services/cerebras.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c58234019..998066fba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Noise Suppression. (see https://picovoice.ai/platform/koala/) +- Added `CerebrasLLMService` for Cerebras integration with an OpenAI-compatible + interface. Added foundational example `14k-function-calling-cerebras.py`. + - Pipecat now supports Python 3.13. We had a dependency on the `audioop` package which was deprecated and now removed on Python 3.13. We are now using `audioop-lts` (https://github.com/AbstractUmbra/audioop) to provide the same diff --git a/README.md b/README.md index 6c7965e45..b4366125b 100644 --- a/README.md +++ b/README.md @@ -55,17 +55,17 @@ pip install "pipecat-ai[option,...]" Available options include: -| Category | Services | Install Command Example | -| ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | -| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | -| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | -| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | -| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | +| Category | Services | Install Command Example | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | +| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | +| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | +| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) @@ -223,7 +223,8 @@ Install the ### PyCharm -`ruff` was installed in the `venv` environment described before, now to enable autoformatting on save, go to `File` -> `Settings` -> `Tools` -> `File Watchers` and add a new watcher with the following settings: +`ruff` was installed in the `venv` environment described before, now to enable autoformatting on save, go to `File` -> `Settings` -> `Tools` -> `File Watchers` and add a new watcher with the following settings: + 1. **Name**: `Ruff formatter` 2. **File type**: `Python` 3. **Working directory**: `$ContentRoot$` diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py new file mode 100644 index 000000000..2b75ad366 --- /dev/null +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from openai.types.chat import ChatCompletionToolParam +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.cerebras import CerebrasLLMService +from pipecat.services.openai import OpenAILLMContext +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def start_fetch_weather(function_name, llm, context): + # note: we can't push a frame to the LLM here. the bot + # can interrupt itself and/or cause audio overlapping glitches. + # possible question for Aleix and Chad about what the right way + # to trigger speech is, now, with the new queues/async/sync refactors. + # await llm.push_frame(TextFrame("Let me check on that.")) + logger.debug(f"Starting fetch_weather_from_api with function_name: {function_name}") + + +async def fetch_weather_from_api(function_name, tool_call_id, args, llm, context, result_callback): + await result_callback({"conditions": "nice", "temperature": "75"}) + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = CerebrasLLMService(api_key=os.getenv("CEREBRAS_API_KEY"), model="llama-3.3-70b") + # Register a function_name of None to get all functions + # sent to the same callback with an additional function_name parameter. + llm.register_function(None, fetch_weather_from_api, start_callback=start_fetch_weather) + + tools = [ + ChatCompletionToolParam( + type="function", + function={ + "name": "get_current_weather", + "description": "Get the current weather for a specific location. You MUST use this function whenever asked about weather.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "format": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The temperature unit to use. Use fahrenheit for US locations, celsius for others.", + }, + }, + "required": ["location", "format"], + }, + }, + ) + ] + messages = [ + { + "role": "system", + "content": """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. + +You have one functions available: + +1. get_current_weather is used to get current weather information. + +Infer whether to use Fahrenheit or Celsius automatically based on the location, unless the user specifies a preference. + +Start by asking me for my location. Then, use 'get_weather_current' to give me a forecast. + + Respond to what the user said in a creative and helpful way.""", + }, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), + context_aggregator.user(), + llm, + tts, + transport.output(), + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index be9eab02f..75889d51c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ aws = [ "boto3~=1.35.27" ] azure = [ "azure-cognitiveservices-speech~=1.41.1", "openai~=1.57.2" ] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.0.13", "websockets~=13.1" ] +cerebras = [ "openai~=1.57.2" ] daily = [ "daily-python~=0.13.0" ] deepgram = [ "deepgram-sdk~=3.7.7" ] elevenlabs = [ "websockets~=13.1" ] diff --git a/src/pipecat/services/cerebras.py b/src/pipecat/services/cerebras.py new file mode 100644 index 000000000..a89b17990 --- /dev/null +++ b/src/pipecat/services/cerebras.py @@ -0,0 +1,38 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +from loguru import logger + +from pipecat.services.openai import OpenAILLMService + + +class CerebrasLLMService(OpenAILLMService): + """A service for interacting with Cerebras's API using the OpenAI-compatible interface. + + This service extends OpenAILLMService to connect to Cerebras's API endpoint while + maintaining full compatibility with OpenAI's interface and functionality. + + Args: + api_key (str): The API key for accessing Cerebras's API + base_url (str, optional): The base URL for Cerebras API. Defaults to "https://api.cerebras.ai/v1" + model (str, optional): The model identifier to use. Defaults to "llama-3.3-70b" + **kwargs: Additional keyword arguments passed to OpenAILLMService + """ + + def __init__( + self, + *, + api_key: str, + base_url: str = "https://api.cerebras.ai/v1", + model: str = "llama-3.3-70b", + **kwargs, + ): + super().__init__(api_key=api_key, base_url=base_url, model=model, **kwargs) + + def create_client(self, api_key=None, base_url=None, **kwargs): + """Create OpenAI-compatible client for Cerebras API endpoint.""" + logger.debug(f"Creating Cerebras client with api {base_url}") + return super().create_client(api_key, base_url, **kwargs) From c9dd9060576438ec1b5621e2cf84ae282633c25d Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 18 Dec 2024 12:21:55 -0500 Subject: [PATCH 60/90] Tailor chat completion inputs to Cerebras API --- src/pipecat/services/cerebras.py | 47 ++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/pipecat/services/cerebras.py b/src/pipecat/services/cerebras.py index a89b17990..a0cc81803 100644 --- a/src/pipecat/services/cerebras.py +++ b/src/pipecat/services/cerebras.py @@ -4,10 +4,25 @@ # SPDX-License-Identifier: BSD 2-Clause License # +from typing import List + from loguru import logger +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.services.openai import OpenAILLMService +try: + from openai import ( + AsyncStream, + ) + from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Fireworks, you need to `pip install pipecat-ai[cerebras]`. Also, set `CEREBRAS_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + class CerebrasLLMService(OpenAILLMService): """A service for interacting with Cerebras's API using the OpenAI-compatible interface. @@ -36,3 +51,35 @@ class CerebrasLLMService(OpenAILLMService): """Create OpenAI-compatible client for Cerebras API endpoint.""" logger.debug(f"Creating Cerebras client with api {base_url}") return super().create_client(api_key, base_url, **kwargs) + + async def get_chat_completions( + self, context: OpenAILLMContext, messages: List[ChatCompletionMessageParam] + ) -> AsyncStream[ChatCompletionChunk]: + """Create a streaming chat completion using Cerebras's API. + + Args: + context (OpenAILLMContext): The context object containing tools configuration + and other settings for the chat completion. + messages (List[ChatCompletionMessageParam]): The list of messages comprising + the conversation history and current request. + + Returns: + AsyncStream[ChatCompletionChunk]: A streaming response of chat completion + chunks that can be processed asynchronously. + """ + params = { + "model": self.model_name, + "stream": True, + "messages": messages, + "tools": context.tools, + "tool_choice": context.tool_choice, + "seed": self._settings["seed"], + "temperature": self._settings["temperature"], + "top_p": self._settings["top_p"], + "max_completion_tokens": self._settings["max_completion_tokens"], + } + + params.update(self._settings["extra"]) + + chunks = await self._client.chat.completions.create(**params) + return chunks From b7b8e59e9ea0a950ce9e611fcb882f511b91c1d5 Mon Sep 17 00:00:00 2001 From: Louis Jordan Date: Thu, 19 Dec 2024 16:57:17 +0000 Subject: [PATCH 61/90] feat: set auto_mode=true - ElevenLabs tts WSS --- src/pipecat/services/elevenlabs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 011c64308..e5310003b 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -312,7 +312,7 @@ class ElevenLabsTTSService(WordTTSService): voice_id = self._voice_id model = self.model_name output_format = self._settings["output_format"] - url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}" + url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode=true" if self._settings["optimize_streaming_latency"]: url += f"&optimize_streaming_latency={self._settings['optimize_streaming_latency']}" From 5341739ecef9e974c87cf867c8dae0d26a9914a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Tue, 17 Dec 2024 16:07:30 -0800 Subject: [PATCH 62/90] transport(base output): avoid pushing EndFrame twice --- src/pipecat/transports/base_output.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index fbe337d6d..81471dd37 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -242,8 +242,7 @@ class BaseOutputTransport(FrameProcessor): await self._set_camera_images(frame.images) elif isinstance(frame, TransportMessageFrame): await self.send_message(frame) - # We will push EndFrame later. - elif not isinstance(frame, EndFrame): + else: await self.push_frame(frame) async def _sink_clock_task_handler(self): @@ -324,6 +323,10 @@ class BaseOutputTransport(FrameProcessor): await self.push_frame(BotSpeakingFrame()) await self.push_frame(BotSpeakingFrame(), FrameDirection.UPSTREAM) + # No need to push EndFrame, it's pushed from process_frame(). + if isinstance(frame, EndFrame): + break + # Handle frame. await self._sink_frame_handler(frame) @@ -333,9 +336,6 @@ class BaseOutputTransport(FrameProcessor): # Send audio. if isinstance(frame, OutputAudioRawFrame): await self.write_raw_audio_frames(frame.audio) - - if isinstance(frame, EndFrame): - break except asyncio.CancelledError: pass except Exception as e: From 8b496f8c6f2204c1bfea37b96a4aa3c14bc26ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 19 Dec 2024 15:27:40 -0800 Subject: [PATCH 63/90] transports(daily): daily-python 0.14.0 (SIP transfer/refer, DTMF) --- CHANGELOG.md | 9 +++++++++ pyproject.toml | 2 +- src/pipecat/transports/services/daily.py | 24 ++++++++++++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14ba01347..526775b16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `DailyTransport.send_dtmf()` to send dial-out DTMF tones. + +- Added `DailyTransport.sip_call_transfer()` to forward SIP and PSTN calls to + another address or number. For example, transfer a SIP call to a different + SIP address or transfer a PSTN phone number to a different PSTN phone number. + +- Added `DailyTransport.sip_refer()` to transfer incoming SIP/PSTN calls from + outside Daily to another SIP/PSTN address. + - Added `KoalaFilter` which implement on device noise reduction using Koala Noise Suppression. (see https://picovoice.ai/platform/koala/) diff --git a/pyproject.toml b/pyproject.toml index 75889d51c..85db84a84 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ azure = [ "azure-cognitiveservices-speech~=1.41.1", "openai~=1.57.2" ] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.0.13", "websockets~=13.1" ] cerebras = [ "openai~=1.57.2" ] -daily = [ "daily-python~=0.13.0" ] +daily = [ "daily-python~=0.14.0" ] deepgram = [ "deepgram-sdk~=3.7.7" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.4.1" ] diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 7456ef816..c0e8a8dd8 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -491,6 +491,21 @@ class DailyTransportClient(EventHandler): self._client.stop_dialout(participant_id, completion=completion_callback(future)) await future + async def send_dtmf(self, settings): + future = self._loop.create_future() + self._client.send_dtmf(settings, completion=completion_callback(future)) + await future + + async def sip_call_transfer(self, settings): + future = self._loop.create_future() + self._client.sip_call_transfer(settings, completion=completion_callback(future)) + await future + + async def sip_refer(self, settings): + future = self._loop.create_future() + self._client.sip_refer(settings, completion=completion_callback(future)) + await future + async def start_recording(self, streaming_settings, stream_id, force_new): future = self._loop.create_future() self._client.start_recording( @@ -955,6 +970,15 @@ class DailyTransport(BaseTransport): async def stop_dialout(self, participant_id): await self._client.stop_dialout(participant_id) + async def send_dtmf(self, settings): + await self._client.send_dtmf(settings) + + async def sip_call_transfer(self, settings): + await self._client.sip_call_transfer(settings) + + async def sip_refer(self, settings): + await self._client.sip_refer(settings) + async def start_recording(self, streaming_settings=None, stream_id=None, force_new=None): await self._client.start_recording(streaming_settings, stream_id, force_new) From 656cbc35e1233681224d1d05dadb5ae8574a0e61 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 19 Dec 2024 12:11:07 -0500 Subject: [PATCH 64/90] Make auto_mode an input parametere for ElevenLabsTTSService; add changelog entry --- CHANGELOG.md | 10 +++++++++- src/pipecat/services/elevenlabs.py | 4 +++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 526775b16..5bdd1cd67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added `DailyTransport.send_dtmf()` to send dial-out DTMF tones. - Added `DailyTransport.sip_call_transfer()` to forward SIP and PSTN calls to - another address or number. For example, transfer a SIP call to a different + another address or number. For example, transfer a SIP call to a different SIP address or transfer a PSTN phone number to a different PSTN phone number. - Added `DailyTransport.sip_refer()` to transfer incoming SIP/PSTN calls from outside Daily to another SIP/PSTN address. +- Added an `auto_mode` input parameter to `ElevenLabsTTSService`. `auto_mode` + is set to `True` by default. Enabling this setting disables the chunk + schedule and all buffers, which reduces latency. + - Added `KoalaFilter` which implement on device noise reduction using Koala Noise Suppression. (see https://picovoice.ai/platform/koala/) @@ -51,6 +55,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `auto_mode: True` for the `ElevenLabsTTSService`. This change + significantly reduces the latency by disabling the chunk schedule and all + buffers. + - `OpenAIRealtimeBetaLLMService` now takes a `model` parameter in the constructor. diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index e5310003b..cffa90025 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -135,6 +135,7 @@ class ElevenLabsTTSService(WordTTSService): similarity_boost: Optional[float] = None style: Optional[float] = None use_speaker_boost: Optional[bool] = None + auto_mode: Optional[bool] = True @model_validator(mode="after") def validate_voice_settings(self): @@ -193,6 +194,7 @@ class ElevenLabsTTSService(WordTTSService): "similarity_boost": params.similarity_boost, "style": params.style, "use_speaker_boost": params.use_speaker_boost, + "auto_mode": str(params.auto_mode).lower(), } self.set_model_name(model) self.set_voice(voice_id) @@ -312,7 +314,7 @@ class ElevenLabsTTSService(WordTTSService): voice_id = self._voice_id model = self.model_name output_format = self._settings["output_format"] - url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode=true" + url = f"{self._url}/v1/text-to-speech/{voice_id}/stream-input?model_id={model}&output_format={output_format}&auto_mode={self._settings['auto_mode']}" if self._settings["optimize_streaming_latency"]: url += f"&optimize_streaming_latency={self._settings['optimize_streaming_latency']}" From 9554804a498b851081bd26d89af03b686e151fff Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 19 Dec 2024 14:28:36 -0500 Subject: [PATCH 65/90] Update 11L default model, allow language to be used by more models --- CHANGELOG.md | 4 +--- src/pipecat/services/elevenlabs.py | 14 ++++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bdd1cd67..0622d4c3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,9 +55,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- `auto_mode: True` for the `ElevenLabsTTSService`. This change - significantly reduces the latency by disabling the chunk schedule and all - buffers. +- The default model for `ElevenLabsTTSService` is now `eleven_flash_v2_5`. - `OpenAIRealtimeBetaLLMService` now takes a `model` parameter in the constructor. diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index cffa90025..65135642d 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -43,6 +43,12 @@ except ModuleNotFoundError as e: ElevenLabsOutputFormat = Literal["pcm_16000", "pcm_22050", "pcm_24000", "pcm_44100"] +ELEVENLABS_MULTILINGUAL_MODELS = { + "eleven_turbo_v2_5", + "eleven_multilingual_v2", + "eleven_flash_v2_5", +} + def language_to_elevenlabs_language(language: Language) -> str | None: BASE_LANGUAGES = { @@ -152,7 +158,7 @@ class ElevenLabsTTSService(WordTTSService): *, api_key: str, voice_id: str, - model: str = "eleven_turbo_v2_5", + model: str = "eleven_flash_v2_5", url: str = "wss://api.elevenlabs.io", output_format: ElevenLabsOutputFormat = "pcm_24000", params: InputParams = InputParams(), @@ -319,13 +325,13 @@ class ElevenLabsTTSService(WordTTSService): if self._settings["optimize_streaming_latency"]: url += f"&optimize_streaming_latency={self._settings['optimize_streaming_latency']}" - # Language can only be used with the 'eleven_turbo_v2_5' model + # Language can only be used with the ELEVENLABS_MULTILINGUAL_MODELS language = self._settings["language"] - if model == "eleven_turbo_v2_5": + if model in ELEVENLABS_MULTILINGUAL_MODELS: url += f"&language_code={language}" else: logger.warning( - f"Language code [{language}] not applied. Language codes can only be used with the 'eleven_turbo_v2_5' model." + f"Language code [{language}] not applied. Language codes can only be used with multilingual models: {', '.join(sorted(ELEVENLABS_MULTILINGUAL_MODELS))}" ) self._websocket = await websockets.connect(url) From 4547609ffb937ea954c4b890c80355c5f44b08c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Thu, 19 Dec 2024 17:49:27 -0800 Subject: [PATCH 66/90] examples(01a): remove unused import --- examples/foundational/01a-local-audio.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index ca7802b1a..5bb8bcbd3 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -8,7 +8,6 @@ import asyncio import os import sys -import aiohttp from dotenv import load_dotenv from loguru import logger From dcf6b6e12010760daa8b4720f7aed91ea87f6cb5 Mon Sep 17 00:00:00 2001 From: marcus-daily <111281783+marcus-daily@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:32:44 +0000 Subject: [PATCH 67/90] Add an RTVIProcessor to the simple-chatbot pipeline --- examples/simple-chatbot/server/bot-gemini.py | 10 ++++++++++ examples/simple-chatbot/server/bot-openai.py | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index 991df1cd1..c233f46dd 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -42,6 +42,8 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import ( + RTVIProcessor, + RTVIConfig, RTVIBotTranscriptionProcessor, RTVIMetricsProcessor, RTVISpeakingProcessor, @@ -179,9 +181,13 @@ async def main(): # This will send `metrics` messages. rtvi_metrics = RTVIMetricsProcessor() + # Handles RTVI messages from the client + rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + pipeline = Pipeline( [ transport.input(), + rtvi, context_aggregator.user(), llm, rtvi_speaking, @@ -204,6 +210,10 @@ async def main(): ) await task.queue_frame(quiet_frame) + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + await rtvi.set_bot_ready() + @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): await transport.capture_participant_transcription(participant["id"]) diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index a3a68c839..a328bfb3a 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -43,10 +43,12 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import ( + RTVIProcessor, RTVIBotTranscriptionProcessor, RTVIMetricsProcessor, RTVISpeakingProcessor, RTVIUserTranscriptionProcessor, + RTVIConfig, ) from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService @@ -201,9 +203,13 @@ async def main(): # This will send `metrics` messages. rtvi_metrics = RTVIMetricsProcessor() + # Handles RTVI messages from the client + rtvi = RTVIProcessor(config=RTVIConfig(config=[])) + pipeline = Pipeline( [ transport.input(), + rtvi, rtvi_speaking, rtvi_user_transcription, context_aggregator.user(), @@ -227,6 +233,10 @@ async def main(): ) await task.queue_frame(quiet_frame) + @rtvi.event_handler("on_client_ready") + async def on_client_ready(rtvi): + await rtvi.set_bot_ready() + @transport.event_handler("on_first_participant_joined") async def on_first_participant_joined(transport, participant): await transport.capture_participant_transcription(participant["id"]) From 41d07692cadf9888b65a3b89425db1ef519b05a4 Mon Sep 17 00:00:00 2001 From: marcus-daily <111281783+marcus-daily@users.noreply.github.com> Date: Fri, 20 Dec 2024 13:41:55 +0000 Subject: [PATCH 68/90] Fix import order --- examples/simple-chatbot/server/bot-gemini.py | 4 ++-- examples/simple-chatbot/server/bot-openai.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index c233f46dd..f06d5d768 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -42,10 +42,10 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import ( - RTVIProcessor, - RTVIConfig, RTVIBotTranscriptionProcessor, + RTVIConfig, RTVIMetricsProcessor, + RTVIProcessor, RTVISpeakingProcessor, RTVIUserTranscriptionProcessor, ) diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index a328bfb3a..932c77b9a 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -43,12 +43,12 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.processors.frameworks.rtvi import ( - RTVIProcessor, RTVIBotTranscriptionProcessor, + RTVIConfig, RTVIMetricsProcessor, + RTVIProcessor, RTVISpeakingProcessor, RTVIUserTranscriptionProcessor, - RTVIConfig, ) from pipecat.services.elevenlabs import ElevenLabsTTSService from pipecat.services.openai import OpenAILLMService From 900b95eb9286dd8b00346ae3c6ec16d40cc26ca0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 20 Dec 2024 09:56:53 -0500 Subject: [PATCH 69/90] Update PlayHT to use the latest Websocket connection endpoint --- CHANGELOG.md | 3 +++ src/pipecat/services/playht.py | 28 +++++++++++++++++++++------- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0622d4c3a..12f631da4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `PlayHTTTSService` uses the new v4 websocket API, which also fixes an issue + where text inputted to the TTS didn't return audio. + - The default model for `ElevenLabsTTSService` is now `eleven_flash_v2_5`. - `OpenAIRealtimeBetaLLMService` now takes a `model` parameter in the diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 43e1739c9..115198c02 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -209,7 +209,7 @@ class PlayHTTTSService(TTSService): async def _get_websocket_url(self): async with aiohttp.ClientSession() as session: async with session.post( - "https://api.play.ht/api/v3/websocket-auth", + "https://api.play.ht/api/v4/websocket-auth", headers={ "Authorization": f"Bearer {self._api_key}", "X-User-Id": self._user_id, @@ -218,10 +218,19 @@ class PlayHTTTSService(TTSService): ) as response: if response.status in (200, 201): data = await response.json() - if "websocket_url" in data and isinstance(data["websocket_url"], str): - self._websocket_url = data["websocket_url"] + # Handle the new response format with multiple URLs + if "websocket_urls" in data: + # Select URL based on voice_engine + if self._settings["voice_engine"] in data["websocket_urls"]: + self._websocket_url = data["websocket_urls"][ + self._settings["voice_engine"] + ] + else: + raise ValueError( + f"Unsupported voice engine: {self._settings['voice_engine']}" + ) else: - raise ValueError("Invalid or missing WebSocket URL in response") + raise ValueError("Invalid response: missing websocket_urls") else: raise Exception(f"Failed to get WebSocket URL: {response.status}") @@ -248,9 +257,14 @@ class PlayHTTTSService(TTSService): logger.debug(f"Received text message: {message}") try: msg = json.loads(message) - if "request_id" in msg and msg["request_id"] == self._request_id: - await self.push_frame(TTSStoppedFrame()) - self._request_id = None + if msg.get("type") == "start": + # Handle start of stream + logger.debug(f"Started processing request: {msg.get('request_id')}") + elif msg.get("type") == "end": + # Handle end of stream + if "request_id" in msg and msg["request_id"] == self._request_id: + await self.push_frame(TTSStoppedFrame()) + self._request_id = None elif "error" in msg: logger.error(f"{self} error: {msg}") await self.push_error(ErrorFrame(f'{self} error: {msg["error"]}')) From 43759295cc9f10721dbb659a27068d51e82a55ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Fri, 20 Dec 2024 09:33:20 -0800 Subject: [PATCH 70/90] frame_processor: reset input queue flag with interruptions --- CHANGELOG.md | 3 +++ src/pipecat/processors/frame_processor.py | 1 + 2 files changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 526775b16..93877bb5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed an issue that could cause the bot to stop talking if there was a user + interruption before getting any audio from the TTS service. + - Fixed an issue that would cause `ParallelPipeline` to handle `EndFrame` incorrectly causing the main pipeline to not terminate or terminate too early. diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index 52066b4f4..e3dfe0bce 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -268,6 +268,7 @@ class FrameProcessor: raise def __create_input_task(self): + self.__should_block_frames = False self.__input_queue = asyncio.Queue() self.__input_frame_task = self.get_event_loop().create_task( self.__input_frame_task_handler() From dac4468ca177f773eb96e03224cf8a50e9224019 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Fri, 20 Dec 2024 14:41:57 -0500 Subject: [PATCH 71/90] Add Fish Audio TTS service --- .../foundational/07t-interruptible-fish.py | 99 ++++++++ pyproject.toml | 1 + src/pipecat/services/fish.py | 234 ++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 examples/foundational/07t-interruptible-fish.py create mode 100644 src/pipecat/services/fish.py diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py new file mode 100644 index 000000000..e710e25c3 --- /dev/null +++ b/examples/foundational/07t-interruptible-fish.py @@ -0,0 +1,99 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import os +import sys + +import aiohttp +from dotenv import load_dotenv +from loguru import logger +from runner import configure + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMMessagesFrame +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.fish import FishAudioTTSService +from pipecat.services.openai import OpenAILLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + + +async def main(): + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = FishAudioTTSService( + api_key=os.getenv("FISH_API_KEY"), + model="4ce7e917cedd4bc2bb2e6ff3a46acaa1", # Barack Obama + ) + + llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") + + messages = [ + { + "role": "system", + "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.", + }, + ] + + context = OpenAILLMContext(messages) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), # User responses + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses + ] + ) + + task = PipelineTask( + pipeline, + PipelineParams( + allow_interruptions=True, + enable_metrics=True, + enable_usage_metrics=True, + report_only_initial_ttfb=True, + ), + ) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + await transport.capture_participant_transcription(participant["id"]) + # Kick off the conversation. + messages.append({"role": "system", "content": "Please introduce yourself to the user."}) + await task.queue_frames([LLMMessagesFrame(messages)]) + + runner = PipelineRunner() + + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 85db84a84..dae895ded 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ daily = [ "daily-python~=0.14.0" ] deepgram = [ "deepgram-sdk~=3.7.7" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.4.1" ] +fish = [ "ormsgpack~=1.7.0", "websockets~=13.1" ] gladia = [ "websockets~=13.1" ] google = [ "google-generativeai~=0.8.3", "google-cloud-texttospeech~=2.21.1" ] grok = [ "openai~=1.57.2" ] diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py new file mode 100644 index 000000000..4fbdee714 --- /dev/null +++ b/src/pipecat/services/fish.py @@ -0,0 +1,234 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import uuid +from typing import AsyncGenerator, Literal, Optional + +from loguru import logger +from pydantic import BaseModel +from tenacity import AsyncRetrying, RetryCallState, stop_after_attempt, wait_exponential + +from pipecat.frames.frames import ( + BotStoppedSpeakingFrame, + CancelFrame, + EndFrame, + ErrorFrame, + Frame, + LLMFullResponseEndFrame, + StartFrame, + StartInterruptionFrame, + TTSAudioRawFrame, + TTSSpeakFrame, + TTSStartedFrame, + TTSStoppedFrame, +) +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.ai_services import TTSService +from pipecat.transcriptions.language import Language + +try: + import ormsgpack + import websockets +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error( + "In order to use Fish Audio, you need to `pip install pipecat-ai[fish]`. Also, set `FISH_API_KEY` environment variable." + ) + raise Exception(f"Missing module: {e}") + +# FishAudio supports various output formats +FishAudioOutputFormat = Literal["opus", "mp3", "pcm", "wav"] + + +class FishAudioTTSService(TTSService): + class InputParams(BaseModel): + language: Optional[Language] = Language.EN + latency: Optional[str] = "normal" # "normal" or "balanced" + prosody_speed: Optional[float] = 1.0 # Speech speed (0.5-2.0) + prosody_volume: Optional[int] = 0 # Volume adjustment in dB + + def __init__( + self, + *, + api_key: str, + model: str, # This is the reference_id + output_format: FishAudioOutputFormat = "pcm", + sample_rate: int = 24000, + params: InputParams = InputParams(), + **kwargs, + ): + super().__init__(sample_rate=sample_rate, **kwargs) + + self._api_key = api_key + self._base_url = "wss://api.fish.audio/v1/tts/live" + self._websocket = None + self._receive_task = None + self._request_id = None + self._started = False + + self._settings = { + "sample_rate": sample_rate, + "latency": params.latency, + "format": output_format, + "prosody": { + "speed": params.prosody_speed, + "volume": params.prosody_volume, + }, + "reference_id": model, + } + + self.set_model_name(model) + + def can_generate_metrics(self) -> bool: + return True + + async def set_model(self, model: str): + self._settings["reference_id"] = model + await super().set_model(model) + logger.info(f"Switching TTS model to: [{model}]") + + async def start(self, frame: StartFrame): + await super().start(frame) + await self._connect() + + async def stop(self, frame: EndFrame): + await super().stop(frame) + await self._disconnect() + + async def cancel(self, frame: CancelFrame): + await super().cancel(frame) + await self._disconnect() + + async def _connect(self): + await self._connect_websocket() + self._receive_task = self.get_event_loop().create_task(self._receive_task_handler()) + + async def _disconnect(self): + await self._disconnect_websocket() + if self._receive_task: + self._receive_task.cancel() + await self._receive_task + self._receive_task = None + + async def _connect_websocket(self): + try: + logger.debug("Connecting to Fish Audio") + headers = {"Authorization": f"Bearer {self._api_key}"} + self._websocket = await websockets.connect(self._base_url, extra_headers=headers) + + # Send initial start message with ormsgpack + start_message = {"event": "start", "request": {"text": "", **self._settings}} + await self._websocket.send(ormsgpack.packb(start_message)) + logger.debug("Sent start message to Fish Audio") + except Exception as e: + logger.error(f"Fish Audio initialization error: {e}") + self._websocket = None + + async def _disconnect_websocket(self): + try: + await self.stop_all_metrics() + if self._websocket: + logger.debug("Disconnecting from Fish Audio") + # Send stop event with ormsgpack + stop_message = {"event": "stop"} + await self._websocket.send(ormsgpack.packb(stop_message)) + await self._websocket.close() + self._websocket = None + self._request_id = None + self._started = False + except Exception as e: + logger.error(f"Error closing websocket: {e}") + + def _get_websocket(self): + if self._websocket: + return self._websocket + raise Exception("Websocket not connected") + + async def _receive_messages(self): + async for message in self._get_websocket(): + try: + if isinstance(message, bytes): + msg = ormsgpack.unpackb(message) + if isinstance(msg, dict): + event = msg.get("event") + print(f"Received event: {event}") + if event == "audio": + await self.stop_ttfb_metrics() + audio_data = msg.get("audio") + # Only process larger chunks to remove msgpack overhead + if audio_data and len(audio_data) > 1024: + frame = TTSAudioRawFrame( + audio_data, self._settings["sample_rate"], 1 + ) + await self.push_frame(frame) + continue + + except Exception as e: + logger.error(f"Error processing message: {e}") + + async def _reconnect_websocket(self, retry_state: RetryCallState): + logger.warning(f"Fish Audio reconnecting (attempt: {retry_state.attempt_number})") + await self._disconnect_websocket() + await self._connect_websocket() + + async def _receive_task_handler(self): + while True: + try: + async for attempt in AsyncRetrying( + stop=stop_after_attempt(3), + wait=wait_exponential(multiplier=1, min=4, max=10), + before_sleep=self._reconnect_websocket, + reraise=True, + ): + with attempt: + await self._receive_messages() + except asyncio.CancelledError: + break + except Exception as e: + message = f"Fish Audio error receiving messages: {e}" + logger.error(message) + await self.push_error(ErrorFrame(message, fatal=True)) + break + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + if isinstance(frame, TTSSpeakFrame): + await self.pause_processing_frames() + elif isinstance(frame, LLMFullResponseEndFrame) and self._request_id: + await self.pause_processing_frames() + elif isinstance(frame, BotStoppedSpeakingFrame): + await self.resume_processing_frames() + + async def _handle_interruption(self, frame: StartInterruptionFrame, direction: FrameDirection): + await super()._handle_interruption(frame, direction) + await self.stop_all_metrics() + self._request_id = None + + async def run_tts(self, text: str) -> AsyncGenerator[Frame, None]: + logger.debug(f"Generating Fish TTS: [{text}]") + try: + if not self._websocket or self._websocket.closed: + await self._connect() + + if not self._request_id: + await self.start_ttfb_metrics() + yield TTSStartedFrame() + self._request_id = str(uuid.uuid4()) + + text_message = { + "event": "text", + "text": text, + } + await self._get_websocket().send(ormsgpack.packb(text_message)) + await self.start_tts_usage_metrics(text) + + yield None + + except Exception as e: + logger.error(f"Error generating TTS: {e}") + yield ErrorFrame(f"Error: {str(e)}") From 627c91f4a67b5579a30f7b7a2bc07d0030ab7534 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 21 Dec 2024 12:42:47 -0500 Subject: [PATCH 72/90] Flush the audio --- src/pipecat/services/fish.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 4fbdee714..723ba06d1 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -155,7 +155,6 @@ class FishAudioTTSService(TTSService): msg = ormsgpack.unpackb(message) if isinstance(msg, dict): event = msg.get("event") - print(f"Received event: {event}") if event == "audio": await self.stop_ttfb_metrics() audio_data = msg.get("audio") @@ -220,12 +219,23 @@ class FishAudioTTSService(TTSService): yield TTSStartedFrame() self._request_id = str(uuid.uuid4()) + # Send the text text_message = { "event": "text", "text": text, } - await self._get_websocket().send(ormsgpack.packb(text_message)) - await self.start_tts_usage_metrics(text) + try: + await self._get_websocket().send(ormsgpack.packb(text_message)) + await self.start_tts_usage_metrics(text) + + # Send flush event to force audio generation + flush_message = {"event": "flush"} + await self._get_websocket().send(ormsgpack.packb(flush_message)) + except Exception as e: + logger.error(f"{self} error sending message: {e}") + yield TTSStoppedFrame() + await self._disconnect() + await self._connect() yield None From bce218915e67a158a3efdb7fbcce92f38a8d25ff Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 21 Dec 2024 12:54:07 -0500 Subject: [PATCH 73/90] Add Fish to the README --- README.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index b4366125b..2821e8f32 100644 --- a/README.md +++ b/README.md @@ -55,17 +55,17 @@ pip install "pipecat-ai[option,...]" Available options include: -| Category | Services | Install Command Example | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | -| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | -| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | -| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | -| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | -| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | -| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | -| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | -| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | +| Category | Services | Install Command Example | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------- | +| Speech-to-Text | [AssemblyAI](https://docs.pipecat.ai/server/services/stt/assemblyai), [Azure](https://docs.pipecat.ai/server/services/stt/azure), [Deepgram](https://docs.pipecat.ai/server/services/stt/deepgram), [Gladia](https://docs.pipecat.ai/server/services/stt/gladia), [Whisper](https://docs.pipecat.ai/server/services/stt/whisper) | `pip install "pipecat-ai[deepgram]"` | +| LLMs | [Anthropic](https://docs.pipecat.ai/server/services/llm/anthropic), [Azure](https://docs.pipecat.ai/server/services/llm/azure), [Cerebras](https://docs.pipecat.ai/server/services/llm/cerebras), [Fireworks AI](https://docs.pipecat.ai/server/services/llm/fireworks), [Gemini](https://docs.pipecat.ai/server/services/llm/gemini), [Grok](https://docs.pipecat.ai/server/services/llm/grok), [Groq](https://docs.pipecat.ai/server/services/llm/groq), [NVIDIA NIM](https://docs.pipecat.ai/server/services/llm/nim), [Ollama](https://docs.pipecat.ai/server/services/llm/ollama), [OpenAI](https://docs.pipecat.ai/server/services/llm/openai), [Together AI](https://docs.pipecat.ai/server/services/llm/together) | `pip install "pipecat-ai[openai]"` | +| Text-to-Speech | [AWS](https://docs.pipecat.ai/server/services/tts/aws), [Azure](https://docs.pipecat.ai/server/services/tts/azure), [Cartesia](https://docs.pipecat.ai/server/services/tts/cartesia), [Deepgram](https://docs.pipecat.ai/server/services/tts/deepgram), [ElevenLabs](https://docs.pipecat.ai/server/services/tts/elevenlabs), [Fish](https://docs.pipecat.ai/server/services/tts/fish), [Google](https://docs.pipecat.ai/server/services/tts/google), [LMNT](https://docs.pipecat.ai/server/services/tts/lmnt), [OpenAI](https://docs.pipecat.ai/server/services/tts/openai), [PlayHT](https://docs.pipecat.ai/server/services/tts/playht), [Rime](https://docs.pipecat.ai/server/services/tts/rime), [XTTS](https://docs.pipecat.ai/server/services/tts/xtts) | `pip install "pipecat-ai[cartesia]"` | +| Speech-to-Speech | [Gemini Multimodal Live](https://docs.pipecat.ai/server/services/s2s/gemini), [OpenAI Realtime](https://docs.pipecat.ai/server/services/s2s/openai) | `pip install "pipecat-ai[openai]"` | +| Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | +| Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | +| Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | +| Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) From 6fabb7e7d568bc0bd2b785e6c4ea6edb88226f1c Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 21 Dec 2024 13:25:43 -0500 Subject: [PATCH 74/90] Fix metrics calculations --- src/pipecat/services/fish.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index 723ba06d1..cc1302fa9 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -156,7 +156,6 @@ class FishAudioTTSService(TTSService): if isinstance(msg, dict): event = msg.get("event") if event == "audio": - await self.stop_ttfb_metrics() audio_data = msg.get("audio") # Only process larger chunks to remove msgpack overhead if audio_data and len(audio_data) > 1024: @@ -164,6 +163,7 @@ class FishAudioTTSService(TTSService): audio_data, self._settings["sample_rate"], 1 ) await self.push_frame(frame) + await self.stop_ttfb_metrics() continue except Exception as e: @@ -216,6 +216,7 @@ class FishAudioTTSService(TTSService): if not self._request_id: await self.start_ttfb_metrics() + await self.start_tts_usage_metrics(text) yield TTSStartedFrame() self._request_id = str(uuid.uuid4()) From 6c117539850d88c67974e840732ff09af7b4a20f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Sat, 21 Dec 2024 14:04:46 -0500 Subject: [PATCH 75/90] Add the ability to send_prebuilt_chat_message when using the DailyTransport --- src/pipecat/transports/services/daily.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index c0e8a8dd8..c00352d98 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -518,6 +518,16 @@ class DailyTransportClient(EventHandler): self._client.stop_recording(stream_id, completion=completion_callback(future)) await future + async def send_prebuilt_chat_message(self, message: str, user_name: str | None = None): + if not self._joined: + return + + future = self._loop.create_future() + self._client.send_prebuilt_chat_message( + message, user_name=user_name, completion=completion_callback(future) + ) + await future + async def capture_participant_transcription(self, participant_id: str): if not self._params.transcription_enabled: return @@ -985,6 +995,15 @@ class DailyTransport(BaseTransport): async def stop_recording(self, stream_id=None): await self._client.stop_recording(stream_id) + async def send_prebuilt_chat_message(self, message: str, user_name: str | None = None): + """Sends a chat message to Daily's Prebuilt main room. + + Args: + message: The chat message to send + user_name: Optional user name that will appear as sender of the message + """ + await self._client.send_prebuilt_chat_message(message, user_name) + async def capture_participant_transcription(self, participant_id: str): await self._client.capture_participant_transcription(participant_id) From 59240c7b96e17eddeb066c6d1aa907e8ee592548 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 21 Dec 2024 14:35:45 -0800 Subject: [PATCH 76/90] delay gemini multimodal live websocket connect --- examples/foundational/07d-interruptible-elevenlabs.py | 3 ++- src/pipecat/services/gemini_multimodal_live/gemini.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index 8f3895d00..9d91081d8 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -48,6 +48,7 @@ async def main(): tts = ElevenLabsTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), + model="eleven_flash_v2_5", ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") @@ -79,7 +80,7 @@ async def main(): allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, - report_only_initial_ttfb=True, + # report_only_initial_ttfb=True, ), ) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index bf433054a..55467a1cd 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -177,6 +177,7 @@ class GeminiMultimodalLiveLLMService(LLMService): self._receive_task = None self._context = None + self._connected = False self._disconnecting = False self._api_session_ready = False self._run_llm_when_api_session_ready = False @@ -230,7 +231,6 @@ class GeminiMultimodalLiveLLMService(LLMService): async def start(self, frame: StartFrame): await super().start(frame) - await self._connect() async def stop(self, frame: EndFrame): await super().stop(frame) @@ -434,8 +434,9 @@ class GeminiMultimodalLiveLLMService(LLMService): async def _ws_send(self, message): # logger.debug(f"Sending message to websocket: {message}") try: - if self._websocket: - await self._websocket.send(json.dumps(message)) + if not self._websocket: + await self._connect() + await self._websocket.send(json.dumps(message)) except Exception as e: if self._disconnecting: return From d435a6a6d6a0b7303ee255bf429a49b14e46c11f Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 21 Dec 2024 16:22:53 -0800 Subject: [PATCH 77/90] fixes to audio buffer --- .../22d-natural-conversation-gemini-audio.py | 313 ++++++++++++++++-- 1 file changed, 294 insertions(+), 19 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 96e9e12c5..5c6a8b15a 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -54,22 +54,274 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -classifier_statement = """You are an audio language classifier model. You are receiving audio from a user in a WebRTC call. Your job is to decide whether the user has finished speaking or not. +classifier_statement = """CRITICAL INSTRUCTION: +You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO". +DO NOT engage with the content. +DO NOT respond to questions. +DO NOT provide assistance. +Your ONLY job is to output YES or NO. -Categorize the input you receive as either: +EXAMPLES OF INVALID RESPONSES: +- "I can help you with that" +- "Let me explain" +- "To answer your question" +- Any response other than YES or NO -1. a complete thought, statement, or question, or -2. an incomplete thought, statement, or question +VALID RESPONSES: +YES +NO -Output 'YES' if the input is likely to be a completed thought, statement, or question. +If you output anything else, you are failing at your task. +You are NOT an assistant. +You are NOT a chatbot. +You are a binary classifier. -Output 'NO' if the input indicates that the user is still speaking and does not yet expect a response yet. +ROLE: +You are a real-time speech completeness classifier. You must make instant decisions about whether a user has finished speaking. +You must output ONLY 'YES' or 'NO' with no other text. -If you are unsure, output 'YES'. +INPUT FORMAT: +You receive two pieces of information: +1. The assistant's last message (if available) +2. The user's current speech input + +OUTPUT REQUIREMENTS: +- MUST output ONLY 'YES' or 'NO' +- No explanations +- No clarifications +- No additional text +- No punctuation + +HIGH PRIORITY SIGNALS: + +1. Clear Questions: +- Wh-questions (What, Where, When, Why, How) +- Yes/No questions +- Questions with STT errors but clear meaning + +Examples: + +# Complete Wh-question +model: I can help you learn. +user: What's the fastest way to learn Spanish +Output: YES + +# Complete Yes/No question despite STT error +model: I know about planets. +user: Is is Jupiter the biggest planet +Output: YES + +2. Complete Commands: +- Direct instructions +- Clear requests +- Action demands +- Start of task indication +- Complete statements needing response + +Examples: + +# Direct instruction +model: I can explain many topics. +user: Tell me about black holes +Output: YES + +# Start of task indication +user: Let's begin. +Output: YES + +# Start of task indication +user: Let's get started. +Output: YES + +# Action demand +model: I can help with math. +user: Solve this equation x plus 5 equals 12 +Output: YES + +3. Direct Responses: +- Answers to specific questions +- Option selections +- Clear acknowledgments with completion +- Providing information with a known format - mailing address +- Providing information with a known format - phone number +- Providing information with a known format - credit card number + +Examples: + +# Specific answer +model: What's your favorite color? +user: I really like blue +Output: YES + +# Option selection +model: Would you prefer morning or evening? +user: Morning +Output: YES + +# Providing information with a known format - mailing address +model: What's your address? +user: 1234 Main Street +Output: NO + +# Providing information with a known format - mailing address +model: What's your address? +user: 1234 Main Street Irving Texas 75063 +Output: Yes + +# Providing information with a known format - phone number +system: A US phone number has 10 digits. +model: What's your phone number? +user: 41086753 +Output: NO + +# Providing information with a known format - phone number +system: A US phone number has 10 digits. +model: What's your phone number? +user: 4108675309 +Output: Yes + +# Providing information with a known format - phone number +system: A US phone number has 10 digits. +model: What's your phone number? +user: 220 +user: 111 +user: 8775 +Output: Yes + +# Providing information with a known format - credit card number +model: What's your phone number? +user: 5556 +Output: NO + +# Providing information with a known format - phone number +model: What's your phone number? +user: 5556710454680800 +Output: Yes + +MEDIUM PRIORITY SIGNALS: + +1. Speech Pattern Completions: +- Self-corrections reaching completion +- False starts with clear ending +- Topic changes with complete thought +- Mid-sentence completions + +Examples: + +# Self-correction reaching completion +model: What would you like to know? +user: Tell me about... no wait, explain how rainbows form +Output: YES + +# Topic change with complete thought +model: The weather is nice today. +user: Actually can you tell me who invented the telephone +Output: YES + +# Mid-sentence completion +model: Hello I'm ready. +user: What's the capital of? France +Output: YES + +2. Context-Dependent Brief Responses: +- Acknowledgments (okay, sure, alright) +- Agreements (yes, yeah) +- Disagreements (no, nah) +- Confirmations (correct, exactly) + +Examples: + +# Acknowledgment +model: Should we talk about history? +user: Sure +Output: YES + +# Disagreement with completion +model: Is that what you meant? +user: No not really +Output: YES + +LOW PRIORITY SIGNALS: + +1. STT Artifacts (Consider but don't over-weight): +- Repeated words +- Unusual punctuation +- Capitalization errors +- Word insertions/deletions + +Examples: + +# Word repetition but complete +model: I can help with that. +user: What what is the time right now +Output: YES + +# Missing punctuation but complete +model: I can explain that. +user: Please tell me how computers work +Output: YES + +2. Speech Features: +- Filler words (um, uh, like) +- Thinking pauses +- Word repetitions +- Brief hesitations + +Examples: + +# Filler words but complete +model: What would you like to know? +user: Um uh how do airplanes fly +Output: YES + +# Thinking pause but incomplete +model: I can explain anything. +user: Well um I want to know about the +Output: NO + +DECISION RULES: + +1. Return YES if: +- ANY high priority signal shows clear completion +- Medium priority signals combine to show completion +- Meaning is clear despite low priority artifacts + +2. Return NO if: +- No high priority signals present +- Thought clearly trails off +- Multiple incomplete indicators +- User appears mid-formulation + +3. When uncertain: +- If you can understand the intent → YES +- If meaning is unclear → NO +- Always make a binary decision +- Never request clarification + +Examples: + +# Incomplete despite corrections +model: What would you like to know about? +user: Can you tell me about +Output: NO + +# Complete despite multiple artifacts +model: I can help you learn. +user: How do you I mean what's the best way to learn programming +Output: YES + +# Trailing off incomplete +model: I can explain anything. +user: I was wondering if you could tell me why +Output: NO """ conversational_system_message = """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. +If you know that a number string is a phone number from the context of the conversation, say it as a phone number. For example 210-333-4567. + +If you know that a number string is a credit card number, say it as a credit card number. For example 4111-1111-1111-1111. + Please be very concise in your responses. Unless you are explicitly asked to do otherwise, give me the shortest complete answer possible without unnecessary elaboration. Generally you should answer with a single sentence. """ @@ -79,13 +331,15 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): super().__init__(**kwargs) self._notifier = notifier self._audio_frames = [] - self._audio_frames = [] self._start_secs = 0.2 # this should match VAD start_secs (hardcoding for now) - self._user_speaking = False + self._max_buffer_size_secs = 30 + self._user_speaking_vad_state = False + self._user_speaking_utterance_state = False async def reset(self): self._audio_frames = [] - self._user_speaking = False + self._user_speaking_vad_state = False + self._user_speaking_utterance_state = False async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -99,22 +353,42 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): # but let's leave that as an exercise to the reader. :-) return if isinstance(frame, UserStartedSpeakingFrame): - self._user_speaking = True + self._user_speaking_vad_state = True + self._user_speaking_utterance_state = True + elif isinstance(frame, UserStoppedSpeakingFrame): + if self._audio_frames[-1]: + fr = self._audio_frames[-1] + frame_duration = len(fr.audio) / 2 * fr.num_channels / fr.sample_rate + + logger.debug( + f"!!! Frame duration: ({len(fr.audio)}) ({fr.num_channels}) ({fr.sample_rate}) {frame_duration}" + ) + + data = b"".join(frame.audio for frame in self._audio_frames) + logger.debug( + f"Processing audio buffer seconds: ({len(self._audio_frames)}) ({len(data)}) {len(data) / 2 / 16000}" + ) self._user_speaking = False context = GoogleLLMContext() context.set_messages([{"role": "system", "content": classifier_statement}]) context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(OpenAILLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): - if self._user_speaking: - self._audio_frames.append(frame) + # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest + # frames as necessary. + # Use a small buffer size when an utterance is not in progress. Just big enough to backfill the start_secs. + # Use a larger buffer size when an utterance is in progress. + # Assume all audio frames have the same duration. + self._audio_frames.append(frame) + frame_duration = len(frame.audio) / 2 * frame.num_channels / frame.sample_rate + buffer_duration = frame_duration * len(self._audio_frames) + # logger.debug(f"!!! Frame duration: {frame_duration}") + if self._user_speaking_utterance_state: + while buffer_duration > self._max_buffer_size_secs: + self._audio_frames.pop(0) + buffer_duration -= frame_duration else: - # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest - # frames as necessary. Assume all audio frames have the same duration. - self._audio_frames.append(frame) - frame_duration = len(frame.audio) / 16 * frame.num_channels / frame.sample_rate - buffer_duration = frame_duration * len(self._audio_frames) while buffer_duration > self._start_secs: self._audio_frames.pop(0) buffer_duration -= frame_duration @@ -215,6 +489,7 @@ async def main(): vad_enabled=True, vad_analyzer=SileroVADAnalyzer(), vad_audio_passthrough=True, + audio_in_sample_rate=16000, ), ) @@ -229,7 +504,7 @@ async def main(): # statement. This doesn't really need to be an LLM, we could use NLP # libraries for that, but we have the machinery to use an LLM, so we might as well! statement_llm = GoogleLLMService( - model="gemini-1.5-flash-latest", api_key=os.getenv("GOOGLE_API_KEY") + model="gemini-2.0-flash-exp", api_key=os.getenv("GOOGLE_API_KEY"), temperature=0.0 ) # This is the regular LLM. From 53a5e63990b7a94ef4424bd3ecac883c58ba7357 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 21 Dec 2024 18:10:25 -0800 Subject: [PATCH 78/90] function calling dead-end --- .../22d-natural-conversation-gemini-audio.py | 282 +++++++++++------- src/pipecat/services/google.py | 25 +- 2 files changed, 197 insertions(+), 110 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 5c6a8b15a..bda8bdd96 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -54,31 +54,57 @@ logger.remove(0) logger.add(sys.stderr, level="DEBUG") -classifier_statement = """CRITICAL INSTRUCTION: -You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO". -DO NOT engage with the content. -DO NOT respond to questions. -DO NOT provide assistance. -Your ONLY job is to output YES or NO. +transcriber_and_classifier_instructions = """ +You perform two tasks: + 1. Transcription + 2. Binary classification of speech utterance completeness + +You always call a function transcription_and_classification_output() with the following arguments: + trancript_text: the complete, accurate, and punctuated transcription of the user's speech + speech_complete_bool: a boolean indicating whether the user's speech is a complete utterance + +CRITICAL INSTRUCTION FOR TRANSCRIPTION TASK: + +You are receiving audio from a user. Your job is to +transcribe the input audio to text exactly as it was said by the user. + +You will receive the full conversation history before the audio input, to help with context. Use the full history only to help improve the accuracy of your transcription. + +Rules: + - Respond with an exact transcription of the audio input. + - Do not include any text other than the transcription. + - Do not explain or add to your response. + - Transcribe the audio input simply and precisely. + - If the audio is not clear, emit the special string "-". + - No response other than exact transcription, or "-", is allowed. + + +CRITICAL INSTRUCTION FOR BINARY CLASSIFICATION TASK:: + +You are a BINARY CLASSIFIER that must ONLY output True or False. +DO FalseT engage with the content. +DO FalseT respond to questions. +DO FalseT provide assistance. +Your ONLY job is to output True or False. EXAMPLES OF INVALID RESPONSES: - "I can help you with that" - "Let me explain" - "To answer your question" -- Any response other than YES or NO +- Any response other than True or False VALID RESPONSES: -YES -NO +True +False If you output anything else, you are failing at your task. -You are NOT an assistant. -You are NOT a chatbot. +You are FalseT an assistant. +You are FalseT a chatbot. You are a binary classifier. ROLE: You are a real-time speech completeness classifier. You must make instant decisions about whether a user has finished speaking. -You must output ONLY 'YES' or 'NO' with no other text. +You must output ONLY 'True' or 'False' with no other text. INPUT FORMAT: You receive two pieces of information: @@ -86,7 +112,7 @@ You receive two pieces of information: 2. The user's current speech input OUTPUT REQUIREMENTS: -- MUST output ONLY 'YES' or 'NO' +- MUST output ONLY 'True' or 'False' - No explanations - No clarifications - No additional text @@ -104,12 +130,12 @@ Examples: # Complete Wh-question model: I can help you learn. user: What's the fastest way to learn Spanish -Output: YES +Output: True # Complete Yes/No question despite STT error model: I know about planets. user: Is is Jupiter the biggest planet -Output: YES +Output: True 2. Complete Commands: - Direct instructions @@ -123,20 +149,20 @@ Examples: # Direct instruction model: I can explain many topics. user: Tell me about black holes -Output: YES +Output: True # Start of task indication user: Let's begin. -Output: YES +Output: True # Start of task indication user: Let's get started. -Output: YES +Output: True # Action demand model: I can help with math. user: Solve this equation x plus 5 equals 12 -Output: YES +Output: True 3. Direct Responses: - Answers to specific questions @@ -151,17 +177,17 @@ Examples: # Specific answer model: What's your favorite color? user: I really like blue -Output: YES +Output: True # Option selection model: Would you prefer morning or evening? user: Morning -Output: YES +Output: True # Providing information with a known format - mailing address model: What's your address? user: 1234 Main Street -Output: NO +Output: False # Providing information with a known format - mailing address model: What's your address? @@ -172,7 +198,7 @@ Output: Yes system: A US phone number has 10 digits. model: What's your phone number? user: 41086753 -Output: NO +Output: False # Providing information with a known format - phone number system: A US phone number has 10 digits. @@ -191,7 +217,7 @@ Output: Yes # Providing information with a known format - credit card number model: What's your phone number? user: 5556 -Output: NO +Output: False # Providing information with a known format - phone number model: What's your phone number? @@ -211,17 +237,17 @@ Examples: # Self-correction reaching completion model: What would you like to know? user: Tell me about... no wait, explain how rainbows form -Output: YES +Output: True # Topic change with complete thought model: The weather is nice today. user: Actually can you tell me who invented the telephone -Output: YES +Output: True # Mid-sentence completion model: Hello I'm ready. user: What's the capital of? France -Output: YES +Output: True 2. Context-Dependent Brief Responses: - Acknowledgments (okay, sure, alright) @@ -234,12 +260,12 @@ Examples: # Acknowledgment model: Should we talk about history? user: Sure -Output: YES +Output: True # Disagreement with completion model: Is that what you meant? user: No not really -Output: YES +Output: True LOW PRIORITY SIGNALS: @@ -254,12 +280,12 @@ Examples: # Word repetition but complete model: I can help with that. user: What what is the time right now -Output: YES +Output: True # Missing punctuation but complete model: I can explain that. user: Please tell me how computers work -Output: YES +Output: True 2. Speech Features: - Filler words (um, uh, like) @@ -272,29 +298,29 @@ Examples: # Filler words but complete model: What would you like to know? user: Um uh how do airplanes fly -Output: YES +Output: True # Thinking pause but incomplete model: I can explain anything. user: Well um I want to know about the -Output: NO +Output: False DECISION RULES: -1. Return YES if: +1. Return True if: - ANY high priority signal shows clear completion - Medium priority signals combine to show completion - Meaning is clear despite low priority artifacts -2. Return NO if: +2. Return False if: - No high priority signals present - Thought clearly trails off - Multiple incomplete indicators - User appears mid-formulation 3. When uncertain: -- If you can understand the intent → YES -- If meaning is unclear → NO +- If you can understand the intent → True +- If meaning is unclear → False - Always make a binary decision - Never request clarification @@ -303,33 +329,69 @@ Examples: # Incomplete despite corrections model: What would you like to know about? user: Can you tell me about -Output: NO +Output: False # Complete despite multiple artifacts model: I can help you learn. user: How do you I mean what's the best way to learn programming -Output: YES +Output: True # Trailing off incomplete model: I can explain anything. user: I was wondering if you could tell me why -Output: NO +Output: False """ -conversational_system_message = """You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. +conversational_system_message = """You are a helpful assistant participating in a voice converation. -If you know that a number string is a phone number from the context of the conversation, say it as a phone number. For example 210-333-4567. +Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. -If you know that a number string is a credit card number, say it as a credit card number. For example 4111-1111-1111-1111. +If you know that a number string is a phone number from the context of the conversation, write it as a phone number. For example 210-333-4567. -Please be very concise in your responses. Unless you are explicitly asked to do otherwise, give me the shortest complete answer possible without unnecessary elaboration. Generally you should answer with a single sentence. +If you know that a number string is a credit card number, write it as a credit card number. For example 4111-1111-1111-1111. + +Please be very concise in your responses. Unless you are explicitly asked to do otherwise, give me shortest complete answer possible without unnecessary elaboration. Generally you should answer with a single sentence. """ -class StatementJudgeAudioContextAccumulator(FrameProcessor): - def __init__(self, *, notifier: BaseNotifier, **kwargs): +async def transcription_and_classification_output(transcript_text: str, speech_complete_bool: bool): + print(f"TRANSCRIPT: {transcript_text}") + print("------") + print(f"COMPLETE: {speech_complete_bool}") + print("------") + return + + +tx_and_cl_tools = [ + { + "function_declarations": [ + { + "name": "transcription_and_classification_output", + "description": "Deliver the transcription and classification output to an external process.", + "parameters": { + "type": "object", + "properties": { + "transcription_text": { + "type": "string", + "description": "The complete, accurate, and punctuated transcription of the user's speech. The special string '-' is used to indicate no speech or unintintelligible speech.", + }, + "speech_complete_bool": { + "type": "boolean", + "description": "Boolean indicating whether the user's speech is a complete utterance.", + }, + }, + "required": ["transcription_text", "speech_complete_bool"], + }, + }, + ] + } +] + + +class AudioAccumulator(FrameProcessor): + def __init__(self, *, notifier: BaseNotifier = None, **kwargs): super().__init__(**kwargs) - self._notifier = notifier + # self._notifier = notifier self._audio_frames = [] self._start_secs = 0.2 # this should match VAD start_secs (hardcoding for now) self._max_buffer_size_secs = 30 @@ -371,7 +433,9 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): ) self._user_speaking = False context = GoogleLLMContext() - context.set_messages([{"role": "system", "content": classifier_statement}]) + context.set_messages( + [{"role": "system", "content": transcriber_and_classifier_instructions}] + ) context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(OpenAILLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): @@ -396,25 +460,28 @@ class StatementJudgeAudioContextAccumulator(FrameProcessor): await self.push_frame(frame, direction) -class CompletenessCheck(FrameProcessor): - def __init__( - self, notifier: BaseNotifier, audio_accumulator: StatementJudgeAudioContextAccumulator - ): - super().__init__() - self._notifier = notifier - self._audio_accumulator = audio_accumulator +# class ClAndTxContextCreator(FrameProcessor): - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if isinstance(frame, TextFrame) and frame.text.startswith("YES"): - logger.debug("Completeness check YES") - await self.push_frame(UserStoppedSpeakingFrame()) - await self._audio_accumulator.reset() - await self._notifier.notify() - elif isinstance(frame, TextFrame): - if frame.text.strip(): - logger.debug(f"Completeness check NO - '{frame.text}'") +# class CompletenessCheck(FrameProcessor): +# def __init__( +# self, notifier: BaseNotifier, audio_accumulator: StatementJudgeAudioContextAccumulator +# ): +# super().__init__() +# self._notifier = notifier +# self._audio_accumulator = audio_accumulator + +# async def process_frame(self, frame: Frame, direction: FrameDirection): +# await super().process_frame(frame, direction) + +# if isinstance(frame, TextFrame) and frame.text.startswith("True"): +# logger.debug("Completeness check True") +# await self.push_frame(UserStoppedSpeakingFrame()) +# await self._audio_accumulator.reset() +# await self._notifier.notify() +# elif isinstance(frame, TextFrame): +# if frame.text.strip(): +# logger.debug(f"Completeness check False - '{frame.text}'") class OutputGate(FrameProcessor): @@ -493,50 +560,52 @@ async def main(): ), ) - stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) - tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) - # This is the LLM that will be used to detect if the user has finished a - # statement. This doesn't really need to be an LLM, we could use NLP - # libraries for that, but we have the machinery to use an LLM, so we might as well! - statement_llm = GoogleLLMService( - model="gemini-2.0-flash-exp", api_key=os.getenv("GOOGLE_API_KEY"), temperature=0.0 + # This is the LLM that will classify and transcribe user speech. + tx_and_cl_llm = GoogleLLMService( + model="gemini-2.0-flash-exp", + api_key=os.getenv("GOOGLE_API_KEY"), + tools=tx_and_cl_tools, + temperature=0.0, + tool_config={ + "function_calling_config": { + "mode": "ANY", + "allowed_function_names": ["transcription_and_classification_output"], + }, + }, ) - # This is the regular LLM. - llm = GoogleLLMService(model="gemini-1.5-flash-latest", api_key=os.getenv("GOOGLE_API_KEY")) + # This is the regular LLM that responds conversationally. + conversation_llm = GoogleLLMService( + model="gemini-2.0-flash-exp", + api_key=os.getenv("GOOGLE_API_KEY"), + system_instruction=conversational_system_message, + ) - messages = [ - { - "role": "system", - "content": conversational_system_message, - }, - ] + context = OpenAILLMContext() + context_aggregator = conversation_llm.create_context_aggregator(context) - context = OpenAILLMContext(messages) - context_aggregator = llm.create_context_aggregator(context) - - # We have instructed the LLM to return 'YES' if it thinks the user - # completed a sentence. So, if it's 'YES' we will return true in this + # We have instructed the LLM to return 'True' if it thinks the user + # completed a sentence. So, if it's 'True' we will return true in this # predicate which will wake up the notifier. async def wake_check_filter(frame): - return frame.text == "YES" + return frame.text == "True" # This is a notifier that we use to synchronize the two LLMs. notifier = EventNotifier() # This turns the LLM context into an inference request to classify the user's speech # as complete or incomplete. - statement_judge_context_filter = StatementJudgeAudioContextAccumulator(notifier=notifier) + # statement_judge_context_filter = StatementJudgeAudioContextAccumulator(notifier=notifier) # This sends a UserStoppedSpeakingFrame and triggers the notifier event - completeness_check = CompletenessCheck( - notifier=notifier, audio_accumulator=statement_judge_context_filter - ) + # completeness_check = CompletenessCheck( + # notifier=notifier, audio_accumulator=statement_judge_context_filter + # ) # # Notify if the user hasn't said anything. async def user_idle_notifier(frame): @@ -562,6 +631,7 @@ async def main(): pipeline = Pipeline( [ transport.input(), + AudioAccumulator(), ParallelPipeline( [ # Pass everything except UserStoppedSpeaking to the elements after @@ -569,24 +639,24 @@ async def main(): FunctionFilter(filter=block_user_stopped_speaking), ], [ - statement_judge_context_filter, - statement_llm, - completeness_check, - ], - [ - stt, - context_aggregator.user(), - # Block everything except OpenAILLMContextFrame and LLMMessagesFrame - FunctionFilter(filter=pass_only_llm_trigger_frames), - llm, - bot_output_gate, # Buffer all llm/tts output until notified. + # cl_and_tx_context_creator, + tx_and_cl_llm, + # completeness_check, + # context_aggregator.user(), ], + # [ + # # Block everything except OpenAILLMContextFrame and LLMMessagesFrame + # # FunctionFilter(filter=pass_only_llm_trigger_frames), + # audio_input_context_creator, + # llm, + # bot_output_gate, # Buffer all llm/tts output until notified. + # ], ), - tts, - user_idle, - transport.output(), - context_aggregator.assistant(), - ] + # tts, + # user_idle, + # transport.output(), + # context_aggregator.assistant(), + ], ) task = PipelineTask( diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 6bbf1d000..4c5d3c205 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -593,6 +593,8 @@ class GoogleLLMService(LLMService): model: str = "gemini-1.5-flash-latest", params: InputParams = InputParams(), system_instruction: Optional[str] = None, + tools: Optional[List[Dict[str, Any]]] = None, + tool_config: Optional[Dict[str, Any]] = None, **kwargs, ): super().__init__(**kwargs) @@ -607,6 +609,8 @@ class GoogleLLMService(LLMService): "top_p": params.top_p, "extra": params.extra if isinstance(params.extra, dict) else {}, } + self._tools = tools + self._tool_config = tool_config def can_generate_metrics(self) -> bool: return True @@ -625,12 +629,13 @@ class GoogleLLMService(LLMService): try: logger.debug( - f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}" + # f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}" + f"!! Generating chat: {context.get_messages_for_logging()}" ) messages = context.messages if context.system_message and self._system_instruction != context.system_message: - logger.debug(f"System instruction changed: {context.system_message}") + # logger.debug(f"System instruction changed: {context.system_message}") self._system_instruction = context.system_message self._create_client() @@ -649,10 +654,21 @@ class GoogleLLMService(LLMService): generation_config = GenerationConfig(**generation_params) if generation_params else None await self.start_ttfb_metrics() - tools = context.tools if context.tools else [] + tools = [] + if context.tools: + tools = context.tools + elif self._tools: + tools = self._tools + tool_config = None + if self._tool_config: + tool_config = self._tool_config response = await self._client.generate_content_async( - contents=messages, tools=tools, stream=True, generation_config=generation_config + contents=messages, + tools=tools, + stream=True, + generation_config=generation_config, + tool_config=tool_config, ) await self.stop_ttfb_metrics() @@ -671,6 +687,7 @@ class GoogleLLMService(LLMService): if c.text: await self.push_frame(TextFrame(c.text)) elif c.function_call: + logger.debug(f"!!! Function call: {c.function_call}") args = type(c.function_call).to_dict(c.function_call).get("args", {}) await self.call_function( context=context, From f3dd35bfd9bc177dfb520c248886c2b16eb742ee Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 21 Dec 2024 22:18:56 -0800 Subject: [PATCH 79/90] working but needs cleanup --- .../22d-natural-conversation-gemini-audio.py | 351 +++++++++++------- 1 file changed, 208 insertions(+), 143 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index bda8bdd96..fd99ca606 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -10,6 +10,7 @@ import sys import time import aiohttp +import google.ai.generativelanguage as glm from dotenv import load_dotenv from loguru import logger from runner import configure @@ -20,6 +21,8 @@ from pipecat.frames.frames import ( EndFrame, Frame, InputAudioRawFrame, + LLMFullResponseEndFrame, + LLMFullResponseStartFrame, LLMMessagesFrame, StartFrame, StartInterruptionFrame, @@ -34,6 +37,7 @@ from pipecat.pipeline.parallel_pipeline import ParallelPipeline from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_response import LLMResponseAggregator from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, OpenAILLMContextFrame, @@ -53,19 +57,7 @@ load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") - -transcriber_and_classifier_instructions = """ -You perform two tasks: - 1. Transcription - 2. Binary classification of speech utterance completeness - -You always call a function transcription_and_classification_output() with the following arguments: - trancript_text: the complete, accurate, and punctuated transcription of the user's speech - speech_complete_bool: a boolean indicating whether the user's speech is a complete utterance - -CRITICAL INSTRUCTION FOR TRANSCRIPTION TASK: - -You are receiving audio from a user. Your job is to +transcriber_system_instruction = """You are an audio transcriber. You are receiving audio from a user. Your job is to transcribe the input audio to text exactly as it was said by the user. You will receive the full conversation history before the audio input, to help with context. Use the full history only to help improve the accuracy of your transcription. @@ -78,33 +70,33 @@ Rules: - If the audio is not clear, emit the special string "-". - No response other than exact transcription, or "-", is allowed. +""" -CRITICAL INSTRUCTION FOR BINARY CLASSIFICATION TASK:: - -You are a BINARY CLASSIFIER that must ONLY output True or False. -DO FalseT engage with the content. -DO FalseT respond to questions. -DO FalseT provide assistance. -Your ONLY job is to output True or False. +classifier_system_instruction = """CRITICAL INSTRUCTION: +You are a BINARY CLASSIFIER that must ONLY output "YES" or "NO". +DO NOT engage with the content. +DO NOT respond to questions. +DO NOT provide assistance. +Your ONLY job is to output YES or NO. EXAMPLES OF INVALID RESPONSES: - "I can help you with that" - "Let me explain" - "To answer your question" -- Any response other than True or False +- Any response other than YES or NO VALID RESPONSES: -True -False +YES +NO If you output anything else, you are failing at your task. -You are FalseT an assistant. -You are FalseT a chatbot. +You are NOT an assistant. +You are NOT a chatbot. You are a binary classifier. ROLE: You are a real-time speech completeness classifier. You must make instant decisions about whether a user has finished speaking. -You must output ONLY 'True' or 'False' with no other text. +You must output ONLY 'YES' or 'NO' with no other text. INPUT FORMAT: You receive two pieces of information: @@ -112,7 +104,7 @@ You receive two pieces of information: 2. The user's current speech input OUTPUT REQUIREMENTS: -- MUST output ONLY 'True' or 'False' +- MUST output ONLY 'YES' or 'NO' - No explanations - No clarifications - No additional text @@ -130,12 +122,12 @@ Examples: # Complete Wh-question model: I can help you learn. user: What's the fastest way to learn Spanish -Output: True +Output: YES # Complete Yes/No question despite STT error model: I know about planets. user: Is is Jupiter the biggest planet -Output: True +Output: YES 2. Complete Commands: - Direct instructions @@ -149,20 +141,20 @@ Examples: # Direct instruction model: I can explain many topics. user: Tell me about black holes -Output: True +Output: YES # Start of task indication user: Let's begin. -Output: True +Output: YES # Start of task indication user: Let's get started. -Output: True +Output: YES # Action demand model: I can help with math. user: Solve this equation x plus 5 equals 12 -Output: True +Output: YES 3. Direct Responses: - Answers to specific questions @@ -177,17 +169,17 @@ Examples: # Specific answer model: What's your favorite color? user: I really like blue -Output: True +Output: YES # Option selection model: Would you prefer morning or evening? user: Morning -Output: True +Output: YES # Providing information with a known format - mailing address model: What's your address? user: 1234 Main Street -Output: False +Output: NO # Providing information with a known format - mailing address model: What's your address? @@ -198,7 +190,7 @@ Output: Yes system: A US phone number has 10 digits. model: What's your phone number? user: 41086753 -Output: False +Output: NO # Providing information with a known format - phone number system: A US phone number has 10 digits. @@ -217,7 +209,7 @@ Output: Yes # Providing information with a known format - credit card number model: What's your phone number? user: 5556 -Output: False +Output: NO # Providing information with a known format - phone number model: What's your phone number? @@ -237,17 +229,17 @@ Examples: # Self-correction reaching completion model: What would you like to know? user: Tell me about... no wait, explain how rainbows form -Output: True +Output: YES # Topic change with complete thought model: The weather is nice today. user: Actually can you tell me who invented the telephone -Output: True +Output: YES # Mid-sentence completion model: Hello I'm ready. user: What's the capital of? France -Output: True +Output: YES 2. Context-Dependent Brief Responses: - Acknowledgments (okay, sure, alright) @@ -260,12 +252,12 @@ Examples: # Acknowledgment model: Should we talk about history? user: Sure -Output: True +Output: YES # Disagreement with completion model: Is that what you meant? user: No not really -Output: True +Output: YES LOW PRIORITY SIGNALS: @@ -280,12 +272,12 @@ Examples: # Word repetition but complete model: I can help with that. user: What what is the time right now -Output: True +Output: YES # Missing punctuation but complete model: I can explain that. user: Please tell me how computers work -Output: True +Output: YES 2. Speech Features: - Filler words (um, uh, like) @@ -298,29 +290,29 @@ Examples: # Filler words but complete model: What would you like to know? user: Um uh how do airplanes fly -Output: True +Output: YES # Thinking pause but incomplete model: I can explain anything. user: Well um I want to know about the -Output: False +Output: NO DECISION RULES: -1. Return True if: +1. Return YES if: - ANY high priority signal shows clear completion - Medium priority signals combine to show completion - Meaning is clear despite low priority artifacts -2. Return False if: +2. Return NO if: - No high priority signals present - Thought clearly trails off - Multiple incomplete indicators - User appears mid-formulation 3. When uncertain: -- If you can understand the intent → True -- If meaning is unclear → False +- If you can understand the intent → YES +- If meaning is unclear → NO - Always make a binary decision - Never request clarification @@ -329,20 +321,20 @@ Examples: # Incomplete despite corrections model: What would you like to know about? user: Can you tell me about -Output: False +Output: NO # Complete despite multiple artifacts model: I can help you learn. user: How do you I mean what's the best way to learn programming -Output: True +Output: YES # Trailing off incomplete model: I can explain anything. user: I was wondering if you could tell me why -Output: False +Output: NO """ -conversational_system_message = """You are a helpful assistant participating in a voice converation. +conversation_system_instruction = """You are a helpful assistant participating in a voice converation. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way. @@ -354,44 +346,9 @@ Please be very concise in your responses. Unless you are explicitly asked to do """ -async def transcription_and_classification_output(transcript_text: str, speech_complete_bool: bool): - print(f"TRANSCRIPT: {transcript_text}") - print("------") - print(f"COMPLETE: {speech_complete_bool}") - print("------") - return - - -tx_and_cl_tools = [ - { - "function_declarations": [ - { - "name": "transcription_and_classification_output", - "description": "Deliver the transcription and classification output to an external process.", - "parameters": { - "type": "object", - "properties": { - "transcription_text": { - "type": "string", - "description": "The complete, accurate, and punctuated transcription of the user's speech. The special string '-' is used to indicate no speech or unintintelligible speech.", - }, - "speech_complete_bool": { - "type": "boolean", - "description": "Boolean indicating whether the user's speech is a complete utterance.", - }, - }, - "required": ["transcription_text", "speech_complete_bool"], - }, - }, - ] - } -] - - class AudioAccumulator(FrameProcessor): - def __init__(self, *, notifier: BaseNotifier = None, **kwargs): + def __init__(self, **kwargs): super().__init__(**kwargs) - # self._notifier = notifier self._audio_frames = [] self._start_secs = 0.2 # this should match VAD start_secs (hardcoding for now) self._max_buffer_size_secs = 30 @@ -433,10 +390,9 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = GoogleLLMContext() - context.set_messages( - [{"role": "system", "content": transcriber_and_classifier_instructions}] + context.add_audio_frames_message( + text="Audio to process", audio_frames=self._audio_frames ) - context.add_audio_frames_message(audio_frames=self._audio_frames) await self.push_frame(OpenAILLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest @@ -463,33 +419,49 @@ class AudioAccumulator(FrameProcessor): # class ClAndTxContextCreator(FrameProcessor): -# class CompletenessCheck(FrameProcessor): -# def __init__( -# self, notifier: BaseNotifier, audio_accumulator: StatementJudgeAudioContextAccumulator -# ): -# super().__init__() -# self._notifier = notifier -# self._audio_accumulator = audio_accumulator +class CompletenessCheck(FrameProcessor): + def __init__(self, notifier: BaseNotifier, audio_accumulator: AudioAccumulator, **kwargs): + super().__init__() + self._notifier = notifier + self._audio_accumulator = audio_accumulator -# async def process_frame(self, frame: Frame, direction: FrameDirection): -# await super().process_frame(frame, direction) + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) -# if isinstance(frame, TextFrame) and frame.text.startswith("True"): -# logger.debug("Completeness check True") -# await self.push_frame(UserStoppedSpeakingFrame()) -# await self._audio_accumulator.reset() -# await self._notifier.notify() -# elif isinstance(frame, TextFrame): -# if frame.text.strip(): -# logger.debug(f"Completeness check False - '{frame.text}'") + if isinstance(frame, TextFrame) and frame.text.startswith("YES"): + logger.debug("Completeness check YES") + await self.push_frame(UserStoppedSpeakingFrame()) + await self._audio_accumulator.reset() + await self._notifier.notify() + elif isinstance(frame, TextFrame): + if frame.text.strip(): + logger.debug(f"Completeness check NO - '{frame.text}'") + + +class TempPrinter(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + if not isinstance(frame, InputAudioRawFrame): + logger.debug(f"!!! {frame}") + await self.push_frame(frame, direction) class OutputGate(FrameProcessor): - def __init__(self, notifier: BaseNotifier, **kwargs): + def __init__( + self, + notifier: BaseNotifier, + context: OpenAILLMContext, + user_transcription_buffer: "UserAggregatorBuffer", + **kwargs, + ): super().__init__(**kwargs) self._gate_open = False self._frames_buffer = [] self._notifier = notifier + self._context = context + self._transcription_buffer = user_transcription_buffer + + logger.debug("!!! OutputGate created") def close_gate(self): self._gate_open = False @@ -524,6 +496,7 @@ class OutputGate(FrameProcessor): self._frames_buffer.append((frame, direction)) async def _start(self): + logger.debug("!!! OutputGate start") self._frames_buffer = [] self._gate_task = self.get_event_loop().create_task(self._gate_task_handler()) @@ -533,14 +506,86 @@ class OutputGate(FrameProcessor): async def _gate_task_handler(self): while True: + logger.debug("!!! Waiting for notifier") try: await self._notifier.wait() + logger.debug("!!! Notified") + transcription = await self._transcription_buffer.wait_for_transcription() + + # logger.debug(f"!!! OutputGate got transcription: {transcription}") + # logger.debug( + # f"!!! OutputGate has messages: {self._context.get_messages_for_logging()}" + # ) + + last_message = self._context.messages[-1] + if last_message.role == "user": + last_message.parts = [glm.Part(text=transcription)] + + # logger.debug( + # f"!!! NOW OutputGate has messages: {self._context.get_messages_for_logging()}" + # ) + self.open_gate() for frame, direction in self._frames_buffer: await self.push_frame(frame, direction) self._frames_buffer = [] except asyncio.CancelledError: break + except Exception as e: + logger.error(f"!!! OutputGate error: {e}") + raise e + break + + +class ConversationAudioContextAssembler(FrameProcessor): + def __init__(self, context: OpenAILLMContext, **kwargs): + super().__init__(**kwargs) + self._context = context + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # We must not block system frames. + if isinstance(frame, SystemFrame): + await self.push_frame(frame, direction) + return + + if isinstance(frame, OpenAILLMContextFrame): + GoogleLLMContext.upgrade_to_google(self._context) + last_message = frame.context.messages[-1] + self._context._messages.append(last_message) + logger.debug( + f"!!! ConversationAudioContextAssembler {self._context.get_messages_for_logging()}" + ) + await self.push_frame(OpenAILLMContextFrame(context=self._context)) + + +class UserAggregatorBuffer(LLMResponseAggregator): + def __init__(self, **kwargs): + super().__init__( + messages=None, + role=None, + start_frame=LLMFullResponseStartFrame, + end_frame=LLMFullResponseEndFrame, + accumulator_frame=TextFrame, + handle_interruptions=True, + expect_stripped_words=False, + ) + self._transcription = "" + + async def _push_aggregation(self): + if self._aggregation: + self._transcription = self._aggregation + self._aggregation = "" + + logger.debug(f"!!! UserAggregatorBuffer: {self._transcription}") + + async def wait_for_transcription(self): + while not self._transcription: + await asyncio.sleep(0.01) + tx = self._transcription + self._transcription = "" + return tx async def main(): @@ -565,25 +610,30 @@ async def main(): voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady ) - # This is the LLM that will classify and transcribe user speech. - tx_and_cl_llm = GoogleLLMService( + # This is the LLM that will transcribe user speech. + tx_llm = GoogleLLMService( + name="Transcriber", model="gemini-2.0-flash-exp", api_key=os.getenv("GOOGLE_API_KEY"), - tools=tx_and_cl_tools, temperature=0.0, - tool_config={ - "function_calling_config": { - "mode": "ANY", - "allowed_function_names": ["transcription_and_classification_output"], - }, - }, + system_instruction=transcriber_system_instruction, + ) + + # This is the LLM that will classify user speech as complete or incomplete. + classifier_llm = GoogleLLMService( + name="Classifier", + model="gemini-2.0-flash-exp", + api_key=os.getenv("GOOGLE_API_KEY"), + temperature=0.0, + system_instruction=classifier_system_instruction, ) # This is the regular LLM that responds conversationally. conversation_llm = GoogleLLMService( + name="Conversation", model="gemini-2.0-flash-exp", api_key=os.getenv("GOOGLE_API_KEY"), - system_instruction=conversational_system_message, + system_instruction=conversation_system_instruction, ) context = OpenAILLMContext() @@ -602,10 +652,11 @@ async def main(): # as complete or incomplete. # statement_judge_context_filter = StatementJudgeAudioContextAccumulator(notifier=notifier) + audio_accumulater = AudioAccumulator() # This sends a UserStoppedSpeakingFrame and triggers the notifier event - # completeness_check = CompletenessCheck( - # notifier=notifier, audio_accumulator=statement_judge_context_filter - # ) + completeness_check = CompletenessCheck( + notifier=notifier, audio_accumulator=audio_accumulater + ) # # Notify if the user hasn't said anything. async def user_idle_notifier(frame): @@ -615,8 +666,6 @@ async def main(): # sentence, this will wake up the notifier if that happens. user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=5.0) - bot_output_gate = OutputGate(notifier=notifier) - async def block_user_stopped_speaking(frame): return not isinstance(frame, UserStoppedSpeakingFrame) @@ -628,10 +677,18 @@ async def main(): or isinstance(frame, StopInterruptionFrame) ) + conversation_audio_context_assembler = ConversationAudioContextAssembler(context=context) + + user_aggregator_buffer = UserAggregatorBuffer() + + bot_output_gate = OutputGate( + notifier=notifier, context=context, user_transcription_buffer=user_aggregator_buffer + ) + pipeline = Pipeline( [ transport.input(), - AudioAccumulator(), + audio_accumulater, ParallelPipeline( [ # Pass everything except UserStoppedSpeaking to the elements after @@ -639,23 +696,31 @@ async def main(): FunctionFilter(filter=block_user_stopped_speaking), ], [ - # cl_and_tx_context_creator, - tx_and_cl_llm, - # completeness_check, - # context_aggregator.user(), + ParallelPipeline( + [ + classifier_llm, + completeness_check, + ], + [ + tx_llm, + user_aggregator_buffer, + ], + ) + ], + [ + # Block everything except OpenAILLMContextFrame and LLMMessagesFrame + # FunctionFilter(filter=pass_only_llm_trigger_frames), + conversation_audio_context_assembler, + conversation_llm, + bot_output_gate, # buffer output until notified. + # TempPrinter(), ], - # [ - # # Block everything except OpenAILLMContextFrame and LLMMessagesFrame - # # FunctionFilter(filter=pass_only_llm_trigger_frames), - # audio_input_context_creator, - # llm, - # bot_output_gate, # Buffer all llm/tts output until notified. - # ], ), - # tts, - # user_idle, - # transport.output(), - # context_aggregator.assistant(), + # wherefore art thou, user context aggregator? + tts, + user_idle, + transport.output(), + context_aggregator.assistant(), ], ) From f5f0de00e416cdc7947d503d820927d2a262610b Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 21 Dec 2024 23:04:00 -0800 Subject: [PATCH 80/90] still some cleanup to do --- .../22d-natural-conversation-gemini-audio.py | 94 ++++++++----------- src/pipecat/services/google.py | 2 +- 2 files changed, 42 insertions(+), 54 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index fd99ca606..78d1271e6 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -187,35 +187,35 @@ user: 1234 Main Street Irving Texas 75063 Output: Yes # Providing information with a known format - phone number -system: A US phone number has 10 digits. model: What's your phone number? user: 41086753 Output: NO # Providing information with a known format - phone number -system: A US phone number has 10 digits. model: What's your phone number? user: 4108675309 Output: Yes # Providing information with a known format - phone number -system: A US phone number has 10 digits. model: What's your phone number? user: 220 -user: 111 -user: 8775 -Output: Yes +Output: No # Providing information with a known format - credit card number -model: What's your phone number? +model: What's your credit card number? user: 5556 Output: NO # Providing information with a known format - phone number -model: What's your phone number? +model: What's your credit card number? user: 5556710454680800 Output: Yes +model: What's your credit card number? +user: 414067 +Output: NO + + MEDIUM PRIORITY SIGNALS: 1. Speech Pattern Completions: @@ -390,9 +390,7 @@ class AudioAccumulator(FrameProcessor): ) self._user_speaking = False context = GoogleLLMContext() - context.add_audio_frames_message( - text="Audio to process", audio_frames=self._audio_frames - ) + context.add_audio_frames_message(text="Audio follows", audio_frames=self._audio_frames) await self.push_frame(OpenAILLMContextFrame(context=context)) elif isinstance(frame, InputAudioRawFrame): # Append the audio frame to our buffer. Treat the buffer as a ring buffer, dropping the oldest @@ -416,34 +414,50 @@ class AudioAccumulator(FrameProcessor): await self.push_frame(frame, direction) -# class ClAndTxContextCreator(FrameProcessor): - - class CompletenessCheck(FrameProcessor): + wait_time = 5.0 + def __init__(self, notifier: BaseNotifier, audio_accumulator: AudioAccumulator, **kwargs): super().__init__() self._notifier = notifier self._audio_accumulator = audio_accumulator + self._idle_task = None + self._wakeup_time = 0 async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if isinstance(frame, TextFrame) and frame.text.startswith("YES"): logger.debug("Completeness check YES") + if self._idle_task: + logger.debug(f"CompletenessCheck idle wait CANCEL") + self._idle_task.cancel() + self._idle_task = None await self.push_frame(UserStoppedSpeakingFrame()) await self._audio_accumulator.reset() await self._notifier.notify() elif isinstance(frame, TextFrame): if frame.text.strip(): logger.debug(f"Completeness check NO - '{frame.text}'") + # start timer to wake up if necessary + if self._wakeup_time: + self._wakeup_time = time.time() + self.wait_time + else: + logger.debug("CompletenessCheck idle wait START") + self._wakeup_time = time.time() + self.wait_time + self._idle_task = self.get_event_loop().create_task(self._idle_task_handler()) - -class TempPrinter(FrameProcessor): - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - if not isinstance(frame, InputAudioRawFrame): - logger.debug(f"!!! {frame}") - await self.push_frame(frame, direction) + async def _idle_task_handler(self): + try: + while time.time() < self._wakeup_time: + await asyncio.sleep(0.01) + logger.debug(f"CompletenessCheck idle wait OVER") + await self._notifier.notify() + except asyncio.CancelledError: + pass + except Exception as e: + logger.error(f"CompletenessCheck idle wait error: {e}") + raise e class OutputGate(FrameProcessor): @@ -461,8 +475,6 @@ class OutputGate(FrameProcessor): self._context = context self._transcription_buffer = user_transcription_buffer - logger.debug("!!! OutputGate created") - def close_gate(self): self._gate_open = False @@ -496,7 +508,6 @@ class OutputGate(FrameProcessor): self._frames_buffer.append((frame, direction)) async def _start(self): - logger.debug("!!! OutputGate start") self._frames_buffer = [] self._gate_task = self.get_event_loop().create_task(self._gate_task_handler()) @@ -506,25 +517,17 @@ class OutputGate(FrameProcessor): async def _gate_task_handler(self): while True: - logger.debug("!!! Waiting for notifier") + # logger.debug("!!! Waiting for notifier") try: await self._notifier.wait() - logger.debug("!!! Notified") - transcription = await self._transcription_buffer.wait_for_transcription() - # logger.debug(f"!!! OutputGate got transcription: {transcription}") - # logger.debug( - # f"!!! OutputGate has messages: {self._context.get_messages_for_logging()}" - # ) + # logger.debug("!!! Notified") + transcription = await self._transcription_buffer.wait_for_transcription() last_message = self._context.messages[-1] if last_message.role == "user": last_message.parts = [glm.Part(text=transcription)] - # logger.debug( - # f"!!! NOW OutputGate has messages: {self._context.get_messages_for_logging()}" - # ) - self.open_gate() for frame, direction in self._frames_buffer: await self.push_frame(frame, direction) @@ -532,7 +535,7 @@ class OutputGate(FrameProcessor): except asyncio.CancelledError: break except Exception as e: - logger.error(f"!!! OutputGate error: {e}") + logger.error(f"OutputGate error: {e}") raise e break @@ -554,9 +557,6 @@ class ConversationAudioContextAssembler(FrameProcessor): GoogleLLMContext.upgrade_to_google(self._context) last_message = frame.context.messages[-1] self._context._messages.append(last_message) - logger.debug( - f"!!! ConversationAudioContextAssembler {self._context.get_messages_for_logging()}" - ) await self.push_frame(OpenAILLMContextFrame(context=self._context)) @@ -578,7 +578,7 @@ class UserAggregatorBuffer(LLMResponseAggregator): self._transcription = self._aggregation self._aggregation = "" - logger.debug(f"!!! UserAggregatorBuffer: {self._transcription}") + logger.debug(f"[Transcription] {self._transcription}") async def wait_for_transcription(self): while not self._transcription: @@ -658,14 +658,6 @@ async def main(): notifier=notifier, audio_accumulator=audio_accumulater ) - # # Notify if the user hasn't said anything. - async def user_idle_notifier(frame): - await notifier.notify() - - # Sometimes the LLM will fail detecting if a user has completed a - # sentence, this will wake up the notifier if that happens. - user_idle = UserIdleProcessor(callback=user_idle_notifier, timeout=5.0) - async def block_user_stopped_speaking(frame): return not isinstance(frame, UserStoppedSpeakingFrame) @@ -708,17 +700,13 @@ async def main(): ) ], [ - # Block everything except OpenAILLMContextFrame and LLMMessagesFrame - # FunctionFilter(filter=pass_only_llm_trigger_frames), conversation_audio_context_assembler, conversation_llm, - bot_output_gate, # buffer output until notified. + bot_output_gate, # buffer output until notified, then flush frames and update context # TempPrinter(), ], ), - # wherefore art thou, user context aggregator? tts, - user_idle, transport.output(), context_aggregator.assistant(), ], diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 4c5d3c205..091c4df06 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -630,7 +630,7 @@ class GoogleLLMService(LLMService): try: logger.debug( # f"Generating chat: {self._system_instruction} | {context.get_messages_for_logging()}" - f"!! Generating chat: {context.get_messages_for_logging()}" + f"Generating chat: {context.get_messages_for_logging()}" ) messages = context.messages From ab5df1a236a93222a736dbd831cf019dfe88f3ba Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 22 Dec 2024 11:19:02 -0800 Subject: [PATCH 81/90] feature complete gemini audio, transcription, and phrase endpointing demo --- .../22d-natural-conversation-gemini-audio.py | 181 +++++++++++------- src/pipecat/services/google.py | 13 +- 2 files changed, 117 insertions(+), 77 deletions(-) diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 78d1271e6..2c6d21f92 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -57,6 +57,14 @@ load_dotenv(override=True) logger.remove(0) logger.add(sys.stderr, level="DEBUG") +# TRANSCRIBER_MODEL = "gemini-1.5-flash-latest" +# CLASSIFIER_MODEL = "gemini-1.5-flash-latest" +# CONVERSATION_MODEL = "gemini-1.5-flash-latest" + +TRANSCRIBER_MODEL = "gemini-2.0-flash-exp" +CLASSIFIER_MODEL = "gemini-2.0-flash-exp" +CONVERSATION_MODEL = "gemini-2.0-flash-exp" + transcriber_system_instruction = """You are an audio transcriber. You are receiving audio from a user. Your job is to transcribe the input audio to text exactly as it was said by the user. @@ -347,6 +355,11 @@ Please be very concise in your responses. Unless you are explicitly asked to do class AudioAccumulator(FrameProcessor): + """Buffers user audio until the user stops speaking. + + Always pushes a fresh context with a single audio message. + """ + def __init__(self, **kwargs): super().__init__(**kwargs) self._audio_frames = [] @@ -376,14 +389,6 @@ class AudioAccumulator(FrameProcessor): self._user_speaking_utterance_state = True elif isinstance(frame, UserStoppedSpeakingFrame): - if self._audio_frames[-1]: - fr = self._audio_frames[-1] - frame_duration = len(fr.audio) / 2 * fr.num_channels / fr.sample_rate - - logger.debug( - f"!!! Frame duration: ({len(fr.audio)}) ({fr.num_channels}) ({fr.sample_rate}) {frame_duration}" - ) - data = b"".join(frame.audio for frame in self._audio_frames) logger.debug( f"Processing audio buffer seconds: ({len(self._audio_frames)}) ({len(data)}) {len(data) / 2 / 16000}" @@ -415,6 +420,12 @@ class AudioAccumulator(FrameProcessor): class CompletenessCheck(FrameProcessor): + """Checks the result of the classifier LLM to determine if the user has finished speaking. + + Triggers the notifier if the user has finished speaking. Also triggers the notifier if an + idle timeout is reached. + """ + wait_time = 5.0 def __init__(self, notifier: BaseNotifier, audio_accumulator: AudioAccumulator, **kwargs): @@ -427,12 +438,13 @@ class CompletenessCheck(FrameProcessor): async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) - if isinstance(frame, TextFrame) and frame.text.startswith("YES"): + if isinstance(frame, UserStartedSpeakingFrame): + if self._idle_task: + self._idle_task.cancel() + elif isinstance(frame, TextFrame) and frame.text.startswith("YES"): logger.debug("Completeness check YES") if self._idle_task: - logger.debug(f"CompletenessCheck idle wait CANCEL") self._idle_task.cancel() - self._idle_task = None await self.push_frame(UserStoppedSpeakingFrame()) await self._audio_accumulator.reset() await self._notifier.notify() @@ -443,7 +455,7 @@ class CompletenessCheck(FrameProcessor): if self._wakeup_time: self._wakeup_time = time.time() + self.wait_time else: - logger.debug("CompletenessCheck idle wait START") + # logger.debug("!!! CompletenessCheck idle wait START") self._wakeup_time = time.time() + self.wait_time self._idle_task = self.get_event_loop().create_task(self._idle_task_handler()) @@ -451,16 +463,87 @@ class CompletenessCheck(FrameProcessor): try: while time.time() < self._wakeup_time: await asyncio.sleep(0.01) - logger.debug(f"CompletenessCheck idle wait OVER") + # logger.debug(f"!!! CompletenessCheck idle wait OVER") + await self._audio_accumulator.reset() await self._notifier.notify() except asyncio.CancelledError: + # logger.debug(f"!!! CompletenessCheck idle wait CANCEL") pass except Exception as e: logger.error(f"CompletenessCheck idle wait error: {e}") raise e + finally: + # logger.debug(f"!!! CompletenessCheck idle wait FINALLY") + self._wakeup_time = 0 + self._idle_task = None + + +class UserAggregatorBuffer(LLMResponseAggregator): + """Buffers the output of the transcription LLM. Used by the bot output gate.""" + + def __init__(self, **kwargs): + super().__init__( + messages=None, + role=None, + start_frame=LLMFullResponseStartFrame, + end_frame=LLMFullResponseEndFrame, + accumulator_frame=TextFrame, + handle_interruptions=True, + expect_stripped_words=False, + ) + self._transcription = "" + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + # parent method pushes frames + if isinstance(frame, UserStartedSpeakingFrame): + self._transcription = "" + + async def _push_aggregation(self): + if self._aggregation: + self._transcription = self._aggregation + self._aggregation = "" + + logger.debug(f"[Transcription] {self._transcription}") + + async def wait_for_transcription(self): + while not self._transcription: + await asyncio.sleep(0.01) + tx = self._transcription + self._transcription = "" + return tx + + +class ConversationAudioContextAssembler(FrameProcessor): + """Takes the single-message context generated by the AudioAccumulator and adds it to the conversation LLM's context.""" + + def __init__(self, context: OpenAILLMContext, **kwargs): + super().__init__(**kwargs) + self._context = context + + async def process_frame(self, frame: Frame, direction: FrameDirection): + await super().process_frame(frame, direction) + + # We must not block system frames. + if isinstance(frame, SystemFrame): + await self.push_frame(frame, direction) + return + + if isinstance(frame, OpenAILLMContextFrame): + GoogleLLMContext.upgrade_to_google(self._context) + last_message = frame.context.messages[-1] + self._context._messages.append(last_message) + await self.push_frame(OpenAILLMContextFrame(context=self._context)) class OutputGate(FrameProcessor): + """Buffers output frames until the notifier is triggered. + + When the notifier fires, waits until a transcription is ready, then: + 1. Replaces the last user audio message with the transcription. + 2. Flushes the frames buffer. + """ + def __init__( self, notifier: BaseNotifier, @@ -501,6 +584,13 @@ class OutputGate(FrameProcessor): await self.push_frame(frame, direction) return + if isinstance(frame, LLMFullResponseStartFrame): + # Remove the audio message from the context. We will never need it again. + # If the completeness check fails, a new audio message will be appended to the context. + # If the completeness check succeeds, our notifier will fire and we will append the + # transcription to the context. + self._context._messages.pop() + if self._gate_open: await self.push_frame(frame, direction) return @@ -517,16 +607,13 @@ class OutputGate(FrameProcessor): async def _gate_task_handler(self): while True: - # logger.debug("!!! Waiting for notifier") try: await self._notifier.wait() - # logger.debug("!!! Notified") - transcription = await self._transcription_buffer.wait_for_transcription() - - last_message = self._context.messages[-1] - if last_message.role == "user": - last_message.parts = [glm.Part(text=transcription)] + transcription = await self._transcription_buffer.wait_for_transcription() or "-" + self._context._messages.append( + glm.Content(role="user", parts=[glm.Part(text=transcription)]) + ) self.open_gate() for frame, direction in self._frames_buffer: @@ -540,54 +627,6 @@ class OutputGate(FrameProcessor): break -class ConversationAudioContextAssembler(FrameProcessor): - def __init__(self, context: OpenAILLMContext, **kwargs): - super().__init__(**kwargs) - self._context = context - - async def process_frame(self, frame: Frame, direction: FrameDirection): - await super().process_frame(frame, direction) - - # We must not block system frames. - if isinstance(frame, SystemFrame): - await self.push_frame(frame, direction) - return - - if isinstance(frame, OpenAILLMContextFrame): - GoogleLLMContext.upgrade_to_google(self._context) - last_message = frame.context.messages[-1] - self._context._messages.append(last_message) - await self.push_frame(OpenAILLMContextFrame(context=self._context)) - - -class UserAggregatorBuffer(LLMResponseAggregator): - def __init__(self, **kwargs): - super().__init__( - messages=None, - role=None, - start_frame=LLMFullResponseStartFrame, - end_frame=LLMFullResponseEndFrame, - accumulator_frame=TextFrame, - handle_interruptions=True, - expect_stripped_words=False, - ) - self._transcription = "" - - async def _push_aggregation(self): - if self._aggregation: - self._transcription = self._aggregation - self._aggregation = "" - - logger.debug(f"[Transcription] {self._transcription}") - - async def wait_for_transcription(self): - while not self._transcription: - await asyncio.sleep(0.01) - tx = self._transcription - self._transcription = "" - return tx - - async def main(): async with aiohttp.ClientSession() as session: (room_url, _) = await configure(session) @@ -613,7 +652,7 @@ async def main(): # This is the LLM that will transcribe user speech. tx_llm = GoogleLLMService( name="Transcriber", - model="gemini-2.0-flash-exp", + model=TRANSCRIBER_MODEL, api_key=os.getenv("GOOGLE_API_KEY"), temperature=0.0, system_instruction=transcriber_system_instruction, @@ -622,7 +661,7 @@ async def main(): # This is the LLM that will classify user speech as complete or incomplete. classifier_llm = GoogleLLMService( name="Classifier", - model="gemini-2.0-flash-exp", + model=CLASSIFIER_MODEL, api_key=os.getenv("GOOGLE_API_KEY"), temperature=0.0, system_instruction=classifier_system_instruction, @@ -631,7 +670,7 @@ async def main(): # This is the regular LLM that responds conversationally. conversation_llm = GoogleLLMService( name="Conversation", - model="gemini-2.0-flash-exp", + model=CONVERSATION_MODEL, api_key=os.getenv("GOOGLE_API_KEY"), system_instruction=conversation_system_instruction, ) diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index 091c4df06..d724d2776 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -635,7 +635,7 @@ class GoogleLLMService(LLMService): messages = context.messages if context.system_message and self._system_instruction != context.system_message: - # logger.debug(f"System instruction changed: {context.system_message}") + logger.debug(f"System instruction changed: {context.system_message}") self._system_instruction = context.system_message self._create_client() @@ -673,15 +673,16 @@ class GoogleLLMService(LLMService): await self.stop_ttfb_metrics() if response.usage_metadata: + # Use only the prompt token count from the response object prompt_tokens = response.usage_metadata.prompt_token_count - completion_tokens = response.usage_metadata.candidates_token_count - total_tokens = response.usage_metadata.total_token_count + total_tokens = prompt_tokens async for chunk in response: if chunk.usage_metadata: - prompt_tokens += response.usage_metadata.prompt_token_count - completion_tokens += response.usage_metadata.candidates_token_count - total_tokens += response.usage_metadata.total_token_count + # Use only the completion_tokens from the chunks. Prompt tokens are already counted and + # are repeated here. + completion_tokens += chunk.usage_metadata.candidates_token_count + total_tokens += chunk.usage_metadata.candidates_token_count try: for c in chunk.parts: if c.text: From 6901c4fa576d7d40815715f01e168ceef784efa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 22 Dec 2024 12:30:17 -0800 Subject: [PATCH 82/90] pyproject: update daily-python to 0.14.2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 85db84a84..9b753bfa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ azure = [ "azure-cognitiveservices-speech~=1.41.1", "openai~=1.57.2" ] canonical = [ "aiofiles~=24.1.0" ] cartesia = [ "cartesia~=1.0.13", "websockets~=13.1" ] cerebras = [ "openai~=1.57.2" ] -daily = [ "daily-python~=0.14.0" ] +daily = [ "daily-python~=0.14.2" ] deepgram = [ "deepgram-sdk~=3.7.7" ] elevenlabs = [ "websockets~=13.1" ] fal = [ "fal-client~=0.4.1" ] From c04c69df955368f4e281e92605941358bd4da7a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Sun, 22 Dec 2024 14:35:01 -0800 Subject: [PATCH 83/90] transports(base_output): fix duplicate push_frame() --- src/pipecat/transports/base_output.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index 81471dd37..c787405cf 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -242,8 +242,6 @@ class BaseOutputTransport(FrameProcessor): await self._set_camera_images(frame.images) elif isinstance(frame, TransportMessageFrame): await self.send_message(frame) - else: - await self.push_frame(frame) async def _sink_clock_task_handler(self): running = True @@ -262,8 +260,13 @@ class BaseOutputTransport(FrameProcessor): if timestamp > current_time: wait_time = nanoseconds_to_seconds(timestamp - current_time) await asyncio.sleep(wait_time) + + # Handle frame. await self._sink_frame_handler(frame) + # Also, push frame downstream in case anyone else needs it. + await self.push_frame(frame) + self._sink_clock_queue.task_done() except asyncio.CancelledError: break From a5e985094b839baafdd2e3cd5d26cfaa0e074ce4 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 22 Dec 2024 19:45:57 -0800 Subject: [PATCH 84/90] remove stray line --- src/pipecat/services/gemini_multimodal_live/gemini.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 55467a1cd..1fded9e1a 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -177,7 +177,6 @@ class GeminiMultimodalLiveLLMService(LLMService): self._receive_task = None self._context = None - self._connected = False self._disconnecting = False self._api_session_ready = False self._run_llm_when_api_session_ready = False From 1368d3db5ceb489deb94563f6ba588326cf938b3 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Mon, 23 Dec 2024 17:33:59 -0800 Subject: [PATCH 85/90] revert elevenlabs example changes --- examples/foundational/07d-interruptible-elevenlabs.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index 9d91081d8..8f3895d00 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -48,7 +48,6 @@ async def main(): tts = ElevenLabsTTSService( api_key=os.getenv("ELEVENLABS_API_KEY", ""), voice_id=os.getenv("ELEVENLABS_VOICE_ID", ""), - model="eleven_flash_v2_5", ) llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"), model="gpt-4o") @@ -80,7 +79,7 @@ async def main(): allow_interruptions=True, enable_metrics=True, enable_usage_metrics=True, - # report_only_initial_ttfb=True, + report_only_initial_ttfb=True, ), ) From 0a1ce1bb630bb904b30f9f23a4f9f41f3798856e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 23 Dec 2024 19:13:59 -0800 Subject: [PATCH 86/90] update CHANGELOG --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea272f9a1..be3978ba5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Constructor arguments for GoogleLLMService to directly set tools and tool_config. + +- Smart turn detection example (`22d-natural-conversation-gemini-audio.py`) that + leverages Gemini 2.0 capabilities (). + (see https://x.com/kwindla/status/1870974144831275410) + - Added `DailyTransport.send_dtmf()` to send dial-out DTMF tones. - Added `DailyTransport.sip_call_transfer()` to forward SIP and PSTN calls to @@ -75,6 +81,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed token counting in `GoogleLLMService`. Tokens were summed incorrectly + (double-counted in many cases). + - Fixed an issue that could cause the bot to stop talking if there was a user interruption before getting any audio from the TTS service. From 6125ea882d619b8ed3fda7d6f7d23863a108d98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 23 Dec 2024 19:19:39 -0800 Subject: [PATCH 87/90] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2821e8f32..126a671b3 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Available options include: | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | | Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](1https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | | Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) From 3a4994370c22fb66dfcf76178cb52e7cdd10d2fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 23 Dec 2024 19:20:23 -0800 Subject: [PATCH 88/90] update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 126a671b3..6786c36f6 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Available options include: | Transport | [Daily (WebRTC)](https://docs.pipecat.ai/server/services/transport/daily), WebSocket, Local | `pip install "pipecat-ai[daily]"` | | Video | [Tavus](https://docs.pipecat.ai/server/services/video/tavus), [Simli](https://docs.pipecat.ai/server/services/video/simli) | `pip install "pipecat-ai[tavus,simli]"` | | Vision & Image | [Moondream](https://docs.pipecat.ai/server/services/vision/moondream), [fal](https://docs.pipecat.ai/server/services/image-generation/fal) | `pip install "pipecat-ai[moondream]"` | -| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](1https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | +| Audio Processing | [Silero VAD](https://docs.pipecat.ai/server/utilities/audio/silero-vad-analyzer), [Krisp](https://docs.pipecat.ai/server/utilities/audio/krisp-filter), [Koala](https://docs.pipecat.ai/server/utilities/audio/koala-filter), [Noisereduce](https://docs.pipecat.ai/server/utilities/audio/noisereduce-filter) | `pip install "pipecat-ai[silero]"` | | Analytics & Metrics | [Canonical AI](https://docs.pipecat.ai/server/services/analytics/canonical), [Sentry](https://docs.pipecat.ai/server/services/analytics/sentry) | `pip install "pipecat-ai[canonical]"` | 📚 [View full services documentation →](https://docs.pipecat.ai/server/services/supported-services) From e9d275f2700787c01198ce5272583d0ffa0aef9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Mon, 23 Dec 2024 19:38:25 -0800 Subject: [PATCH 89/90] update CHANGELOG for 0.0.52 --- CHANGELOG.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index be3978ba5..5610525b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ All notable changes to **Pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.0.52] - 2024-12-24 ### Added @@ -98,8 +98,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 only be pushed downstream after the audio generated from `TTSSpeakFrame` has been spoken. -- Fixed a [bug](https://github.com/pipecat-ai/pipecat/issues/868) in `DeepgramSTTService` that was causing Language to be passed - as python object instead of a string, causing connection to fail. +- Fixed a `DeepgramSTTService` issue that was causing language to be passed as + an object instead of a string resulting in the connection to fail. ## [0.0.51] - 2024-12-16 From 4667624b60475bb35f9398cc21df3535f897c512 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 6 Jan 2025 10:12:20 -0500 Subject: [PATCH 90/90] Update copyright to 2025 --- LICENSE | 2 +- examples/canonical-metrics/bot.py | 2 +- examples/canonical-metrics/runner.py | 2 +- examples/canonical-metrics/server.py | 2 +- examples/chatbot-audio-recording/bot.py | 2 +- examples/chatbot-audio-recording/runner.py | 2 +- examples/chatbot-audio-recording/server.py | 2 +- .../deployment/flyio-example/bot_runner.py | 2 +- examples/foundational/01-say-one-thing.py | 2 +- examples/foundational/01a-local-audio.py | 2 +- examples/foundational/01c-fastpitch.py | 2 +- examples/foundational/02-llm-say-one-thing.py | 2 +- examples/foundational/03-still-frame.py | 2 +- .../foundational/03a-local-still-frame.py | 2 +- .../foundational/04-utterance-and-speech.py | 2 +- .../foundational/05-sync-speech-and-image.py | 2 +- .../05a-local-sync-speech-and-image.py | 2 +- .../foundational/06-listen-and-respond.py | 2 +- examples/foundational/06a-image-sync.py | 2 +- examples/foundational/07-interruptible-vad.py | 2 +- examples/foundational/07-interruptible.py | 2 +- .../07a-interruptible-anthropic.py | 2 +- .../07b-interruptible-langchain.py | 2 +- .../07c-interruptible-deepgram-vad.py | 2 +- .../07c-interruptible-deepgram.py | 2 +- .../07d-interruptible-elevenlabs.py | 2 +- .../foundational/07e-interruptible-playht.py | 2 +- .../foundational/07f-interruptible-azure.py | 2 +- .../07g-interruptible-openai-tts.py | 2 +- .../07h-interruptible-openpipe.py | 2 +- .../foundational/07i-interruptible-xtts.py | 2 +- .../foundational/07j-interruptible-gladia.py | 2 +- .../foundational/07k-interruptible-lmnt.py | 2 +- .../07l-interruptible-together.py | 2 +- .../foundational/07m-interruptible-polly.py | 2 +- .../foundational/07n-interruptible-google.py | 2 +- .../07o-interruptible-assemblyai.py | 2 +- .../foundational/07p-interruptible-krisp.py | 2 +- .../foundational/07q-interruptible-rime.py | 2 +- .../07r-interruptible-riva-nim.py | 2 +- .../07s-interruptible-google-audio-in.py | 2 +- .../foundational/07t-interruptible-fish.py | 2 +- examples/foundational/09-mirror.py | 2 +- examples/foundational/09a-local-mirror.py | 2 +- examples/foundational/10-wake-phrase.py | 2 +- examples/foundational/11-sound-effects.py | 2 +- examples/foundational/12-describe-video.py | 2 +- .../12a-describe-video-gemini-flash.py | 2 +- .../foundational/12b-describe-video-gpt-4o.py | 2 +- .../12c-describe-video-anthropic.py | 2 +- .../foundational/13-whisper-transcription.py | 2 +- examples/foundational/13a-whisper-local.py | 2 +- .../13b-deepgram-transcription.py | 2 +- .../foundational/13c-gladia-transcription.py | 2 +- .../13d-assemblyai-transcription.py | 2 +- examples/foundational/14-function-calling.py | 2 +- .../14a-function-calling-anthropic.py | 2 +- .../14b-function-calling-anthropic-video.py | 2 +- .../14c-function-calling-together.py | 2 +- .../14d-function-calling-video.py | 2 +- .../14e-function-calling-gemini.py | 2 +- .../foundational/14f-function-calling-groq.py | 2 +- .../foundational/14g-function-calling-grok.py | 2 +- .../14h-function-calling-azure.py | 2 +- .../14i-function-calling-fireworks.py | 2 +- .../foundational/14j-function-calling-nim.py | 2 +- .../14k-function-calling-cerebras.py | 2 +- examples/foundational/15-switch-voices.py | 2 +- examples/foundational/15a-switch-languages.py | 2 +- .../16-gpu-container-local-bot.py | 2 +- examples/foundational/17-detect-user-idle.py | 2 +- examples/foundational/18-gstreamer-filesrc.py | 2 +- .../18a-gstreamer-videotestsrc.py | 2 +- .../foundational/19-openai-realtime-beta.py | 2 +- .../20a-persistent-context-openai.py | 2 +- .../20b-persistent-context-openai-realtime.py | 2 +- .../20c-persistent-context-anthropic.py | 2 +- .../20d-persistent-context-gemini.py | 2 +- examples/foundational/21-tavus-layer.py | 2 +- .../foundational/22-natural-conversation.py | 2 +- .../22b-natural-conversation-proposal.py | 2 +- .../22c-natural-conversation-mixed-llms.py | 2 +- .../22d-natural-conversation-gemini-audio.py | 2 +- .../foundational/23-bot-background-sound.py | 2 +- examples/foundational/24-stt-mute-filter.py | 2 +- examples/foundational/25-google-audio-in.py | 17 ++++++---------- .../foundational/26-gemini-multimodal-live.py | 2 +- ...6a-gemini-multimodal-live-transcription.py | 2 +- ...gemini-multimodal-live-function-calling.py | 2 +- .../26c-gemini-multimodal-live-video.py | 2 +- examples/foundational/27-simli-layer.py | 2 +- .../28a-transcription-processor-openai.py | 2 +- .../28b-transcript-processor-anthropic.py | 2 +- .../28c-transcription-processor-gemini.py | 2 +- examples/foundational/runner.py | 2 +- examples/moondream-chatbot/bot.py | 5 ++--- examples/moondream-chatbot/runner.py | 2 +- examples/moondream-chatbot/server.py | 2 +- examples/patient-intake/bot.py | 2 +- examples/patient-intake/runner.py | 2 +- examples/patient-intake/server.py | 2 +- .../examples/javascript/src/app.js | 2 +- examples/simple-chatbot/server/bot-gemini.py | 2 +- examples/simple-chatbot/server/bot-openai.py | 2 +- examples/simple-chatbot/server/runner.py | 2 +- examples/simple-chatbot/server/server.py | 2 +- .../storytelling-chatbot/src/bot_runner.py | 5 ++--- examples/studypal/runner.py | 2 +- examples/translation-chatbot/bot.py | 2 +- examples/translation-chatbot/runner.py | 2 +- examples/translation-chatbot/server.py | 2 +- examples/websocket-server/bot.py | 2 +- examples/websocket-server/frames.proto | 2 +- .../audio/filters/base_audio_filter.py | 2 +- src/pipecat/audio/filters/koala_filter.py | 2 +- src/pipecat/audio/filters/krisp_filter.py | 5 ++--- .../audio/filters/noisereduce_filter.py | 2 +- src/pipecat/audio/mixers/base_audio_mixer.py | 2 +- src/pipecat/audio/mixers/soundfile_mixer.py | 2 +- src/pipecat/audio/utils.py | 2 +- src/pipecat/audio/vad/silero.py | 2 +- src/pipecat/audio/vad/vad_analyzer.py | 2 +- src/pipecat/clocks/base_clock.py | 2 +- src/pipecat/clocks/system_clock.py | 2 +- src/pipecat/frames/frames.proto | 2 +- src/pipecat/frames/frames.py | 2 +- src/pipecat/pipeline/base_pipeline.py | 2 +- src/pipecat/pipeline/parallel_pipeline.py | 2 +- src/pipecat/pipeline/pipeline.py | 2 +- src/pipecat/pipeline/runner.py | 2 +- .../pipeline/sync_parallel_pipeline.py | 2 +- src/pipecat/pipeline/task.py | 2 +- src/pipecat/processors/aggregators/gated.py | 2 +- .../aggregators/gated_openai_llm_context.py | 2 +- .../processors/aggregators/llm_response.py | 2 +- .../aggregators/openai_llm_context.py | 2 +- .../processors/aggregators/sentence.py | 2 +- .../processors/aggregators/user_response.py | 2 +- .../aggregators/vision_image_frame.py | 2 +- src/pipecat/processors/async_generator.py | 2 +- .../audio/audio_buffer_processor.py | 2 +- src/pipecat/processors/audio/vad/silero.py | 2 +- .../processors/filters/frame_filter.py | 2 +- .../processors/filters/function_filter.py | 2 +- .../processors/filters/identity_filter.py | 2 +- src/pipecat/processors/filters/null_filter.py | 2 +- .../processors/filters/stt_mute_filter.py | 2 +- .../processors/filters/wake_check_filter.py | 5 ++--- .../filters/wake_notifier_filter.py | 2 +- src/pipecat/processors/frame_processor.py | 2 +- .../processors/frameworks/langchain.py | 2 +- src/pipecat/processors/frameworks/rtvi.py | 2 +- .../processors/gstreamer/pipeline_source.py | 2 +- .../processors/idle_frame_processor.py | 2 +- src/pipecat/processors/logger.py | 2 +- .../metrics/frame_processor_metrics.py | 2 +- src/pipecat/processors/metrics/sentry.py | 2 +- src/pipecat/processors/text_transformer.py | 2 +- .../processors/transcript_processor.py | 2 +- src/pipecat/processors/user_idle_processor.py | 2 +- src/pipecat/serializers/base_serializer.py | 2 +- src/pipecat/serializers/livekit.py | 2 +- src/pipecat/serializers/protobuf.py | 7 ++++--- src/pipecat/serializers/twilio.py | 2 +- src/pipecat/services/ai_services.py | 2 +- src/pipecat/services/anthropic.py | 2 +- src/pipecat/services/assemblyai.py | 2 +- src/pipecat/services/aws.py | 2 +- src/pipecat/services/azure.py | 2 +- src/pipecat/services/canonical.py | 3 +-- src/pipecat/services/cartesia.py | 2 +- src/pipecat/services/cerebras.py | 2 +- src/pipecat/services/deepgram.py | 2 +- src/pipecat/services/elevenlabs.py | 2 +- src/pipecat/services/fal.py | 2 +- src/pipecat/services/fireworks.py | 2 +- src/pipecat/services/fish.py | 2 +- .../audio_transcriber.py | 2 +- .../services/gemini_multimodal_live/events.py | 2 +- .../services/gemini_multimodal_live/gemini.py | 2 +- src/pipecat/services/gladia.py | 2 +- src/pipecat/services/google.py | 2 +- src/pipecat/services/grok.py | 2 +- src/pipecat/services/groq.py | 2 +- src/pipecat/services/lmnt.py | 2 +- src/pipecat/services/moondream.py | 6 ++---- src/pipecat/services/nim.py | 2 +- src/pipecat/services/ollama.py | 2 +- src/pipecat/services/openai.py | 2 +- .../services/openai_realtime_beta/context.py | 2 +- .../services/openai_realtime_beta/events.py | 2 +- .../services/openai_realtime_beta/frames.py | 2 +- .../services/openai_realtime_beta/openai.py | 2 +- src/pipecat/services/openpipe.py | 2 +- src/pipecat/services/playht.py | 2 +- src/pipecat/services/rime.py | 2 +- src/pipecat/services/riva.py | 2 +- src/pipecat/services/simli.py | 2 +- src/pipecat/services/tavus.py | 2 +- src/pipecat/services/together.py | 2 +- src/pipecat/services/whisper.py | 5 +++-- src/pipecat/services/xtts.py | 2 +- src/pipecat/sync/base_notifier.py | 2 +- src/pipecat/sync/event_notifier.py | 2 +- src/pipecat/transcriptions/language.py | 2 +- src/pipecat/transports/base_input.py | 2 +- src/pipecat/transports/base_output.py | 2 +- src/pipecat/transports/base_transport.py | 2 +- src/pipecat/transports/local/audio.py | 2 +- src/pipecat/transports/local/tk.py | 2 +- .../transports/network/fastapi_websocket.py | 2 +- .../transports/network/websocket_server.py | 2 +- src/pipecat/transports/services/daily.py | 2 +- .../transports/services/helpers/daily_rest.py | 2 +- src/pipecat/transports/services/livekit.py | 2 +- src/pipecat/utils/string.py | 2 +- src/pipecat/utils/text/base_text_filter.py | 2 +- .../utils/text/markdown_text_filter.py | 20 +++++++------------ src/pipecat/utils/time.py | 2 +- src/pipecat/utils/utils.py | 5 ++--- src/pipecat/vad/silero.py | 2 +- src/pipecat/vad/vad_analyzer.py | 2 +- tests/test_langchain.py | 2 +- 223 files changed, 245 insertions(+), 262 deletions(-) diff --git a/LICENSE b/LICENSE index cd6220df2..a457e0dfa 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ BSD 2-Clause License -Copyright (c) 2024, Daily +Copyright (c) 2025, Daily Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/examples/canonical-metrics/bot.py b/examples/canonical-metrics/bot.py index 71aca70f3..2c2d35911 100644 --- a/examples/canonical-metrics/bot.py +++ b/examples/canonical-metrics/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/canonical-metrics/runner.py b/examples/canonical-metrics/runner.py index a0b46ca36..defe65dfd 100644 --- a/examples/canonical-metrics/runner.py +++ b/examples/canonical-metrics/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/canonical-metrics/server.py b/examples/canonical-metrics/server.py index 2ed6c8239..19cda92a3 100644 --- a/examples/canonical-metrics/server.py +++ b/examples/canonical-metrics/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/chatbot-audio-recording/bot.py b/examples/chatbot-audio-recording/bot.py index ee679ffcf..128ed3ec2 100644 --- a/examples/chatbot-audio-recording/bot.py +++ b/examples/chatbot-audio-recording/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/chatbot-audio-recording/runner.py b/examples/chatbot-audio-recording/runner.py index a0b46ca36..defe65dfd 100644 --- a/examples/chatbot-audio-recording/runner.py +++ b/examples/chatbot-audio-recording/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/chatbot-audio-recording/server.py b/examples/chatbot-audio-recording/server.py index 2ed6c8239..19cda92a3 100644 --- a/examples/chatbot-audio-recording/server.py +++ b/examples/chatbot-audio-recording/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/deployment/flyio-example/bot_runner.py b/examples/deployment/flyio-example/bot_runner.py index fcdfa65ab..d04bcd60f 100644 --- a/examples/deployment/flyio-example/bot_runner.py +++ b/examples/deployment/flyio-example/bot_runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/01-say-one-thing.py b/examples/foundational/01-say-one-thing.py index 290a0b80c..14a89340b 100644 --- a/examples/foundational/01-say-one-thing.py +++ b/examples/foundational/01-say-one-thing.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/01a-local-audio.py b/examples/foundational/01a-local-audio.py index 5bb8bcbd3..7262f3cfe 100644 --- a/examples/foundational/01a-local-audio.py +++ b/examples/foundational/01a-local-audio.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/01c-fastpitch.py b/examples/foundational/01c-fastpitch.py index 01a68abd9..c9ccb2695 100644 --- a/examples/foundational/01c-fastpitch.py +++ b/examples/foundational/01c-fastpitch.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/02-llm-say-one-thing.py b/examples/foundational/02-llm-say-one-thing.py index 1e1ad2827..ec1209fa7 100644 --- a/examples/foundational/02-llm-say-one-thing.py +++ b/examples/foundational/02-llm-say-one-thing.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/03-still-frame.py b/examples/foundational/03-still-frame.py index e532a4b37..310953b1f 100644 --- a/examples/foundational/03-still-frame.py +++ b/examples/foundational/03-still-frame.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/03a-local-still-frame.py b/examples/foundational/03a-local-still-frame.py index 821c9979c..4544e8de5 100644 --- a/examples/foundational/03a-local-still-frame.py +++ b/examples/foundational/03a-local-still-frame.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/04-utterance-and-speech.py b/examples/foundational/04-utterance-and-speech.py index 02f035485..89f941838 100644 --- a/examples/foundational/04-utterance-and-speech.py +++ b/examples/foundational/04-utterance-and-speech.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/05-sync-speech-and-image.py b/examples/foundational/05-sync-speech-and-image.py index 1648b11a7..7990df1c8 100644 --- a/examples/foundational/05-sync-speech-and-image.py +++ b/examples/foundational/05-sync-speech-and-image.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/05a-local-sync-speech-and-image.py b/examples/foundational/05a-local-sync-speech-and-image.py index d62d853cf..e29a20f32 100644 --- a/examples/foundational/05a-local-sync-speech-and-image.py +++ b/examples/foundational/05a-local-sync-speech-and-image.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/06-listen-and-respond.py b/examples/foundational/06-listen-and-respond.py index 5f9bb0a1f..4ccca2792 100644 --- a/examples/foundational/06-listen-and-respond.py +++ b/examples/foundational/06-listen-and-respond.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/06a-image-sync.py b/examples/foundational/06a-image-sync.py index 1406e93a2..853dacf5b 100644 --- a/examples/foundational/06a-image-sync.py +++ b/examples/foundational/06a-image-sync.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07-interruptible-vad.py b/examples/foundational/07-interruptible-vad.py index 53adf84f5..d496b582f 100644 --- a/examples/foundational/07-interruptible-vad.py +++ b/examples/foundational/07-interruptible-vad.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07-interruptible.py b/examples/foundational/07-interruptible.py index c68d4aa40..94fe43c23 100644 --- a/examples/foundational/07-interruptible.py +++ b/examples/foundational/07-interruptible.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07a-interruptible-anthropic.py b/examples/foundational/07a-interruptible-anthropic.py index e7e680eab..52e1bf2fa 100644 --- a/examples/foundational/07a-interruptible-anthropic.py +++ b/examples/foundational/07a-interruptible-anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07b-interruptible-langchain.py b/examples/foundational/07b-interruptible-langchain.py index 8bf4ceea2..3b596cd24 100644 --- a/examples/foundational/07b-interruptible-langchain.py +++ b/examples/foundational/07b-interruptible-langchain.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07c-interruptible-deepgram-vad.py b/examples/foundational/07c-interruptible-deepgram-vad.py index 40866d760..18546759c 100644 --- a/examples/foundational/07c-interruptible-deepgram-vad.py +++ b/examples/foundational/07c-interruptible-deepgram-vad.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07c-interruptible-deepgram.py b/examples/foundational/07c-interruptible-deepgram.py index e52ca370e..0af28ed41 100644 --- a/examples/foundational/07c-interruptible-deepgram.py +++ b/examples/foundational/07c-interruptible-deepgram.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07d-interruptible-elevenlabs.py b/examples/foundational/07d-interruptible-elevenlabs.py index 8f3895d00..ff950acd5 100644 --- a/examples/foundational/07d-interruptible-elevenlabs.py +++ b/examples/foundational/07d-interruptible-elevenlabs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07e-interruptible-playht.py b/examples/foundational/07e-interruptible-playht.py index 293e0b74d..bb3febe68 100644 --- a/examples/foundational/07e-interruptible-playht.py +++ b/examples/foundational/07e-interruptible-playht.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07f-interruptible-azure.py b/examples/foundational/07f-interruptible-azure.py index b08deb1b3..89ff5b7eb 100644 --- a/examples/foundational/07f-interruptible-azure.py +++ b/examples/foundational/07f-interruptible-azure.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07g-interruptible-openai-tts.py b/examples/foundational/07g-interruptible-openai-tts.py index 17dbb2b30..32d537229 100644 --- a/examples/foundational/07g-interruptible-openai-tts.py +++ b/examples/foundational/07g-interruptible-openai-tts.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07h-interruptible-openpipe.py b/examples/foundational/07h-interruptible-openpipe.py index aa4373d4b..deb606e7d 100644 --- a/examples/foundational/07h-interruptible-openpipe.py +++ b/examples/foundational/07h-interruptible-openpipe.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07i-interruptible-xtts.py b/examples/foundational/07i-interruptible-xtts.py index ea20b0238..28ba557f7 100644 --- a/examples/foundational/07i-interruptible-xtts.py +++ b/examples/foundational/07i-interruptible-xtts.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07j-interruptible-gladia.py b/examples/foundational/07j-interruptible-gladia.py index b6866d8af..e2ad5178d 100644 --- a/examples/foundational/07j-interruptible-gladia.py +++ b/examples/foundational/07j-interruptible-gladia.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07k-interruptible-lmnt.py b/examples/foundational/07k-interruptible-lmnt.py index 31b65d6a5..683bb493f 100644 --- a/examples/foundational/07k-interruptible-lmnt.py +++ b/examples/foundational/07k-interruptible-lmnt.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07l-interruptible-together.py b/examples/foundational/07l-interruptible-together.py index 96315623b..3de029c3e 100644 --- a/examples/foundational/07l-interruptible-together.py +++ b/examples/foundational/07l-interruptible-together.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07m-interruptible-polly.py b/examples/foundational/07m-interruptible-polly.py index df376b2f4..22e3059c8 100644 --- a/examples/foundational/07m-interruptible-polly.py +++ b/examples/foundational/07m-interruptible-polly.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07n-interruptible-google.py b/examples/foundational/07n-interruptible-google.py index c0b29715e..b6322d2b9 100644 --- a/examples/foundational/07n-interruptible-google.py +++ b/examples/foundational/07n-interruptible-google.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07o-interruptible-assemblyai.py b/examples/foundational/07o-interruptible-assemblyai.py index 0ae548f9c..49f18aa5e 100644 --- a/examples/foundational/07o-interruptible-assemblyai.py +++ b/examples/foundational/07o-interruptible-assemblyai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07p-interruptible-krisp.py b/examples/foundational/07p-interruptible-krisp.py index 27070df0b..dfc085f8f 100644 --- a/examples/foundational/07p-interruptible-krisp.py +++ b/examples/foundational/07p-interruptible-krisp.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07q-interruptible-rime.py b/examples/foundational/07q-interruptible-rime.py index 2e13e2a8e..7f13be30a 100644 --- a/examples/foundational/07q-interruptible-rime.py +++ b/examples/foundational/07q-interruptible-rime.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07r-interruptible-riva-nim.py b/examples/foundational/07r-interruptible-riva-nim.py index 1e02f82c0..2a8fcce7a 100644 --- a/examples/foundational/07r-interruptible-riva-nim.py +++ b/examples/foundational/07r-interruptible-riva-nim.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07s-interruptible-google-audio-in.py b/examples/foundational/07s-interruptible-google-audio-in.py index 854cf1954..cd250ba07 100644 --- a/examples/foundational/07s-interruptible-google-audio-in.py +++ b/examples/foundational/07s-interruptible-google-audio-in.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/07t-interruptible-fish.py b/examples/foundational/07t-interruptible-fish.py index e710e25c3..c09f67a29 100644 --- a/examples/foundational/07t-interruptible-fish.py +++ b/examples/foundational/07t-interruptible-fish.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/09-mirror.py b/examples/foundational/09-mirror.py index 676935da6..f5a1cf337 100644 --- a/examples/foundational/09-mirror.py +++ b/examples/foundational/09-mirror.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/09a-local-mirror.py b/examples/foundational/09a-local-mirror.py index 50b729883..2ad41ef5f 100644 --- a/examples/foundational/09a-local-mirror.py +++ b/examples/foundational/09a-local-mirror.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/10-wake-phrase.py b/examples/foundational/10-wake-phrase.py index b69ef683b..e943238d3 100644 --- a/examples/foundational/10-wake-phrase.py +++ b/examples/foundational/10-wake-phrase.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/11-sound-effects.py b/examples/foundational/11-sound-effects.py index a52b214e7..9bce4769f 100644 --- a/examples/foundational/11-sound-effects.py +++ b/examples/foundational/11-sound-effects.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/12-describe-video.py b/examples/foundational/12-describe-video.py index 9003ca6f0..0cfd6a729 100644 --- a/examples/foundational/12-describe-video.py +++ b/examples/foundational/12-describe-video.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/12a-describe-video-gemini-flash.py b/examples/foundational/12a-describe-video-gemini-flash.py index df37c24a9..64e168b62 100644 --- a/examples/foundational/12a-describe-video-gemini-flash.py +++ b/examples/foundational/12a-describe-video-gemini-flash.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/12b-describe-video-gpt-4o.py b/examples/foundational/12b-describe-video-gpt-4o.py index 4e96f7264..22a303fee 100644 --- a/examples/foundational/12b-describe-video-gpt-4o.py +++ b/examples/foundational/12b-describe-video-gpt-4o.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/12c-describe-video-anthropic.py b/examples/foundational/12c-describe-video-anthropic.py index 1c88b5dd4..06c35e3ae 100644 --- a/examples/foundational/12c-describe-video-anthropic.py +++ b/examples/foundational/12c-describe-video-anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/13-whisper-transcription.py b/examples/foundational/13-whisper-transcription.py index 72fee4bb0..7ca11522d 100644 --- a/examples/foundational/13-whisper-transcription.py +++ b/examples/foundational/13-whisper-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/13a-whisper-local.py b/examples/foundational/13a-whisper-local.py index dcb5fd51a..88ec65149 100644 --- a/examples/foundational/13a-whisper-local.py +++ b/examples/foundational/13a-whisper-local.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/13b-deepgram-transcription.py b/examples/foundational/13b-deepgram-transcription.py index ad464e87a..e7a6ca335 100644 --- a/examples/foundational/13b-deepgram-transcription.py +++ b/examples/foundational/13b-deepgram-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/13c-gladia-transcription.py b/examples/foundational/13c-gladia-transcription.py index acc21b6c2..06488f094 100644 --- a/examples/foundational/13c-gladia-transcription.py +++ b/examples/foundational/13c-gladia-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/13d-assemblyai-transcription.py b/examples/foundational/13d-assemblyai-transcription.py index d10a80274..db7dba57b 100644 --- a/examples/foundational/13d-assemblyai-transcription.py +++ b/examples/foundational/13d-assemblyai-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14-function-calling.py b/examples/foundational/14-function-calling.py index 0aebc915a..bccc18196 100644 --- a/examples/foundational/14-function-calling.py +++ b/examples/foundational/14-function-calling.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14a-function-calling-anthropic.py b/examples/foundational/14a-function-calling-anthropic.py index 24587c935..04eb75cbb 100644 --- a/examples/foundational/14a-function-calling-anthropic.py +++ b/examples/foundational/14a-function-calling-anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14b-function-calling-anthropic-video.py b/examples/foundational/14b-function-calling-anthropic-video.py index 7473af10d..689d6935b 100644 --- a/examples/foundational/14b-function-calling-anthropic-video.py +++ b/examples/foundational/14b-function-calling-anthropic-video.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14c-function-calling-together.py b/examples/foundational/14c-function-calling-together.py index a629753e5..fc6836147 100644 --- a/examples/foundational/14c-function-calling-together.py +++ b/examples/foundational/14c-function-calling-together.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14d-function-calling-video.py b/examples/foundational/14d-function-calling-video.py index 93a87f420..ce0298ca9 100644 --- a/examples/foundational/14d-function-calling-video.py +++ b/examples/foundational/14d-function-calling-video.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14e-function-calling-gemini.py b/examples/foundational/14e-function-calling-gemini.py index 35b3fa264..34748f39e 100644 --- a/examples/foundational/14e-function-calling-gemini.py +++ b/examples/foundational/14e-function-calling-gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14f-function-calling-groq.py b/examples/foundational/14f-function-calling-groq.py index bcd26672f..8140dc9c4 100644 --- a/examples/foundational/14f-function-calling-groq.py +++ b/examples/foundational/14f-function-calling-groq.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14g-function-calling-grok.py b/examples/foundational/14g-function-calling-grok.py index 0165f1f7e..d83288c24 100644 --- a/examples/foundational/14g-function-calling-grok.py +++ b/examples/foundational/14g-function-calling-grok.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14h-function-calling-azure.py b/examples/foundational/14h-function-calling-azure.py index 29b9ef15a..eb9f3b364 100644 --- a/examples/foundational/14h-function-calling-azure.py +++ b/examples/foundational/14h-function-calling-azure.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14i-function-calling-fireworks.py b/examples/foundational/14i-function-calling-fireworks.py index b3b10df87..462f0107a 100644 --- a/examples/foundational/14i-function-calling-fireworks.py +++ b/examples/foundational/14i-function-calling-fireworks.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14j-function-calling-nim.py b/examples/foundational/14j-function-calling-nim.py index 4059bc386..89e059a5b 100644 --- a/examples/foundational/14j-function-calling-nim.py +++ b/examples/foundational/14j-function-calling-nim.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/14k-function-calling-cerebras.py b/examples/foundational/14k-function-calling-cerebras.py index 2b75ad366..d41266105 100644 --- a/examples/foundational/14k-function-calling-cerebras.py +++ b/examples/foundational/14k-function-calling-cerebras.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/15-switch-voices.py b/examples/foundational/15-switch-voices.py index aa516fdb2..8e1a7ed16 100644 --- a/examples/foundational/15-switch-voices.py +++ b/examples/foundational/15-switch-voices.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/15a-switch-languages.py b/examples/foundational/15a-switch-languages.py index 86798ba50..0e9dc2948 100644 --- a/examples/foundational/15a-switch-languages.py +++ b/examples/foundational/15a-switch-languages.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/16-gpu-container-local-bot.py b/examples/foundational/16-gpu-container-local-bot.py index 85a2ddb0b..686aa01ac 100644 --- a/examples/foundational/16-gpu-container-local-bot.py +++ b/examples/foundational/16-gpu-container-local-bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/17-detect-user-idle.py b/examples/foundational/17-detect-user-idle.py index a3e11fb0f..d0e6eca27 100644 --- a/examples/foundational/17-detect-user-idle.py +++ b/examples/foundational/17-detect-user-idle.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/18-gstreamer-filesrc.py b/examples/foundational/18-gstreamer-filesrc.py index 169e46bc4..c76d8f5b0 100644 --- a/examples/foundational/18-gstreamer-filesrc.py +++ b/examples/foundational/18-gstreamer-filesrc.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/18a-gstreamer-videotestsrc.py b/examples/foundational/18a-gstreamer-videotestsrc.py index 778fea06d..5f91a3a3f 100644 --- a/examples/foundational/18a-gstreamer-videotestsrc.py +++ b/examples/foundational/18a-gstreamer-videotestsrc.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 49ede65be..383160f27 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/20a-persistent-context-openai.py b/examples/foundational/20a-persistent-context-openai.py index bf8af53ea..0c6cd49be 100644 --- a/examples/foundational/20a-persistent-context-openai.py +++ b/examples/foundational/20a-persistent-context-openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index c22eaf128..7398891b0 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/20c-persistent-context-anthropic.py b/examples/foundational/20c-persistent-context-anthropic.py index f8a42fda3..9bbcad278 100644 --- a/examples/foundational/20c-persistent-context-anthropic.py +++ b/examples/foundational/20c-persistent-context-anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/20d-persistent-context-gemini.py b/examples/foundational/20d-persistent-context-gemini.py index 52736606f..faef2b429 100644 --- a/examples/foundational/20d-persistent-context-gemini.py +++ b/examples/foundational/20d-persistent-context-gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/21-tavus-layer.py b/examples/foundational/21-tavus-layer.py index abb2f202d..f985bfba1 100644 --- a/examples/foundational/21-tavus-layer.py +++ b/examples/foundational/21-tavus-layer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/22-natural-conversation.py b/examples/foundational/22-natural-conversation.py index 6f9dbbb5b..dc47e0956 100644 --- a/examples/foundational/22-natural-conversation.py +++ b/examples/foundational/22-natural-conversation.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/22b-natural-conversation-proposal.py b/examples/foundational/22b-natural-conversation-proposal.py index 1ed4360e3..92dd98d4b 100644 --- a/examples/foundational/22b-natural-conversation-proposal.py +++ b/examples/foundational/22b-natural-conversation-proposal.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/22c-natural-conversation-mixed-llms.py b/examples/foundational/22c-natural-conversation-mixed-llms.py index 97bc57ec1..8a3d60c76 100644 --- a/examples/foundational/22c-natural-conversation-mixed-llms.py +++ b/examples/foundational/22c-natural-conversation-mixed-llms.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/22d-natural-conversation-gemini-audio.py b/examples/foundational/22d-natural-conversation-gemini-audio.py index 2c6d21f92..4d618f726 100644 --- a/examples/foundational/22d-natural-conversation-gemini-audio.py +++ b/examples/foundational/22d-natural-conversation-gemini-audio.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/23-bot-background-sound.py b/examples/foundational/23-bot-background-sound.py index cde90f933..a8809648c 100644 --- a/examples/foundational/23-bot-background-sound.py +++ b/examples/foundational/23-bot-background-sound.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/24-stt-mute-filter.py b/examples/foundational/24-stt-mute-filter.py index c1cb33aed..cdd248850 100644 --- a/examples/foundational/24-stt-mute-filter.py +++ b/examples/foundational/24-stt-mute-filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/25-google-audio-in.py b/examples/foundational/25-google-audio-in.py index 477b72ddc..472ca974b 100644 --- a/examples/foundational/25-google-audio-in.py +++ b/examples/foundational/25-google-audio-in.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -80,8 +80,7 @@ Rules: class UserAudioCollector(FrameProcessor): - """ - This FrameProcessor collects audio frames in a buffer, then adds them to the + """This FrameProcessor collects audio frames in a buffer, then adds them to the LLM context when the user stops speaking. """ @@ -125,8 +124,7 @@ class UserAudioCollector(FrameProcessor): class InputTranscriptionContextFilter(FrameProcessor): - """ - This FrameProcessor blocks all frames except the OpenAILLMContextFrame that triggers + """This FrameProcessor blocks all frames except the OpenAILLMContextFrame that triggers LLM inference. (And system frames, which are needed for the pipeline element lifecycle.) We take the context object out of the OpenAILLMContextFrame and use it to create a new @@ -186,8 +184,7 @@ class InputTranscriptionContextFilter(FrameProcessor): @dataclass class LLMDemoTranscriptionFrame(Frame): - """ - It would be nice if we could just use a TranscriptionFrame to send our transcriber + """It would be nice if we could just use a TranscriptionFrame to send our transcriber LLM's transcription output down the pipelline. But we can't, because TranscriptionFrame is a child class of TextFrame, which in our pipeline will be interpreted by the TTS service as text that should be turned into speech. We could restructure this pipeline, @@ -199,8 +196,7 @@ class LLMDemoTranscriptionFrame(Frame): class InputTranscriptionFrameEmitter(FrameProcessor): - """ - A simple FrameProcessor that aggregates the TextFrame output from the transcriber LLM + """A simple FrameProcessor that aggregates the TextFrame output from the transcriber LLM and then sends the full response down the pipeline as an LLMDemoTranscriptionFrame. """ @@ -221,8 +217,7 @@ class InputTranscriptionFrameEmitter(FrameProcessor): class TranscriptionContextFixup(FrameProcessor): - """ - This FrameProcessor looks for the LLMDemoTranscriptionFrame and swaps out the + """This FrameProcessor looks for the LLMDemoTranscriptionFrame and swaps out the audio part of the most recent user message with the text transcription. Audio is big, using a lot of tokens and network bandwidth. So doing this is diff --git a/examples/foundational/26-gemini-multimodal-live.py b/examples/foundational/26-gemini-multimodal-live.py index 3b528ec26..2f0161412 100644 --- a/examples/foundational/26-gemini-multimodal-live.py +++ b/examples/foundational/26-gemini-multimodal-live.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/26a-gemini-multimodal-live-transcription.py b/examples/foundational/26a-gemini-multimodal-live-transcription.py index f44d6d9d2..a4369c800 100644 --- a/examples/foundational/26a-gemini-multimodal-live-transcription.py +++ b/examples/foundational/26a-gemini-multimodal-live-transcription.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/26b-gemini-multimodal-live-function-calling.py b/examples/foundational/26b-gemini-multimodal-live-function-calling.py index 482cc7caf..a11080e92 100644 --- a/examples/foundational/26b-gemini-multimodal-live-function-calling.py +++ b/examples/foundational/26b-gemini-multimodal-live-function-calling.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/26c-gemini-multimodal-live-video.py b/examples/foundational/26c-gemini-multimodal-live-video.py index 50cabc386..3449ed631 100644 --- a/examples/foundational/26c-gemini-multimodal-live-video.py +++ b/examples/foundational/26c-gemini-multimodal-live-video.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/27-simli-layer.py b/examples/foundational/27-simli-layer.py index 415384380..b06c0e4ae 100644 --- a/examples/foundational/27-simli-layer.py +++ b/examples/foundational/27-simli-layer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/28a-transcription-processor-openai.py b/examples/foundational/28a-transcription-processor-openai.py index 0966c882f..4eec1f93e 100644 --- a/examples/foundational/28a-transcription-processor-openai.py +++ b/examples/foundational/28a-transcription-processor-openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/28b-transcript-processor-anthropic.py b/examples/foundational/28b-transcript-processor-anthropic.py index 066828652..5ee6d60ed 100644 --- a/examples/foundational/28b-transcript-processor-anthropic.py +++ b/examples/foundational/28b-transcript-processor-anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/28c-transcription-processor-gemini.py b/examples/foundational/28c-transcription-processor-gemini.py index 6c8118c57..37c0fae43 100644 --- a/examples/foundational/28c-transcription-processor-gemini.py +++ b/examples/foundational/28c-transcription-processor-gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/foundational/runner.py b/examples/foundational/runner.py index f4c774757..55c6a33d3 100644 --- a/examples/foundational/runner.py +++ b/examples/foundational/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/moondream-chatbot/bot.py b/examples/moondream-chatbot/bot.py index a815148b1..aae6c0987 100644 --- a/examples/moondream-chatbot/bot.py +++ b/examples/moondream-chatbot/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -68,8 +68,7 @@ talking_frame = SpriteFrame(images=sprites) class TalkingAnimation(FrameProcessor): - """ - This class starts a talking animation when it receives an first AudioFrame, + """This class starts a talking animation when it receives an first AudioFrame, and then returns to a "quiet" sprite when it sees a TTSStoppedFrame. """ diff --git a/examples/moondream-chatbot/runner.py b/examples/moondream-chatbot/runner.py index f19fcf211..8924e0370 100644 --- a/examples/moondream-chatbot/runner.py +++ b/examples/moondream-chatbot/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/moondream-chatbot/server.py b/examples/moondream-chatbot/server.py index 4caa47f5b..1d2c021be 100644 --- a/examples/moondream-chatbot/server.py +++ b/examples/moondream-chatbot/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/patient-intake/bot.py b/examples/patient-intake/bot.py index 7c65393c7..a7bc6c925 100644 --- a/examples/patient-intake/bot.py +++ b/examples/patient-intake/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/patient-intake/runner.py b/examples/patient-intake/runner.py index f19fcf211..8924e0370 100644 --- a/examples/patient-intake/runner.py +++ b/examples/patient-intake/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/patient-intake/server.py b/examples/patient-intake/server.py index 2d2fee0ed..51b8d95eb 100644 --- a/examples/patient-intake/server.py +++ b/examples/patient-intake/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/simple-chatbot/examples/javascript/src/app.js b/examples/simple-chatbot/examples/javascript/src/app.js index a844453c4..6a32f1fc1 100644 --- a/examples/simple-chatbot/examples/javascript/src/app.js +++ b/examples/simple-chatbot/examples/javascript/src/app.js @@ -1,5 +1,5 @@ /** - * Copyright (c) 2024, Daily + * Copyright (c) 2025, Daily * * SPDX-License-Identifier: BSD 2-Clause License */ diff --git a/examples/simple-chatbot/server/bot-gemini.py b/examples/simple-chatbot/server/bot-gemini.py index f06d5d768..b195d02f1 100644 --- a/examples/simple-chatbot/server/bot-gemini.py +++ b/examples/simple-chatbot/server/bot-gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/simple-chatbot/server/bot-openai.py b/examples/simple-chatbot/server/bot-openai.py index 932c77b9a..90c5e08d8 100644 --- a/examples/simple-chatbot/server/bot-openai.py +++ b/examples/simple-chatbot/server/bot-openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/simple-chatbot/server/runner.py b/examples/simple-chatbot/server/runner.py index 0c03a96bf..ed20d985e 100644 --- a/examples/simple-chatbot/server/runner.py +++ b/examples/simple-chatbot/server/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/simple-chatbot/server/server.py b/examples/simple-chatbot/server/server.py index 7dc6d1f5c..8fb0aa1fc 100644 --- a/examples/simple-chatbot/server/server.py +++ b/examples/simple-chatbot/server/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/storytelling-chatbot/src/bot_runner.py b/examples/storytelling-chatbot/src/bot_runner.py index 79de6c3e3..4c4e0dcbc 100644 --- a/examples/storytelling-chatbot/src/bot_runner.py +++ b/examples/storytelling-chatbot/src/bot_runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -154,8 +154,7 @@ async def catch_all(path_name: Optional[str] = ""): async def virtualize_bot(room_url: str, token: str): - """ - This is an example of how to virtualize the bot using Fly.io + """This is an example of how to virtualize the bot using Fly.io You can adapt this method to use whichever cloud provider you prefer. """ FLY_API_HOST = os.getenv("FLY_API_HOST", "https://api.machines.dev/v1") diff --git a/examples/studypal/runner.py b/examples/studypal/runner.py index f4c774757..55c6a33d3 100644 --- a/examples/studypal/runner.py +++ b/examples/studypal/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/translation-chatbot/bot.py b/examples/translation-chatbot/bot.py index 946864426..1404b3698 100644 --- a/examples/translation-chatbot/bot.py +++ b/examples/translation-chatbot/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/translation-chatbot/runner.py b/examples/translation-chatbot/runner.py index f19fcf211..8924e0370 100644 --- a/examples/translation-chatbot/runner.py +++ b/examples/translation-chatbot/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/translation-chatbot/server.py b/examples/translation-chatbot/server.py index ffd8fc5d6..2f8104738 100644 --- a/examples/translation-chatbot/server.py +++ b/examples/translation-chatbot/server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/websocket-server/bot.py b/examples/websocket-server/bot.py index 80633ac74..fd792d0df 100644 --- a/examples/websocket-server/bot.py +++ b/examples/websocket-server/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/examples/websocket-server/frames.proto b/examples/websocket-server/frames.proto index 4c58d2a34..484042f18 100644 --- a/examples/websocket-server/frames.proto +++ b/examples/websocket-server/frames.proto @@ -1,5 +1,5 @@ // -// Copyright (c) 2024, Daily +// Copyright (c) 2025, Daily // // SPDX-License-Identifier: BSD 2-Clause License // diff --git a/src/pipecat/audio/filters/base_audio_filter.py b/src/pipecat/audio/filters/base_audio_filter.py index e635bb1ba..58ecca33c 100644 --- a/src/pipecat/audio/filters/base_audio_filter.py +++ b/src/pipecat/audio/filters/base_audio_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/filters/koala_filter.py b/src/pipecat/audio/filters/koala_filter.py index 416e4e9fb..ebd0b1ed4 100644 --- a/src/pipecat/audio/filters/koala_filter.py +++ b/src/pipecat/audio/filters/koala_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/filters/krisp_filter.py b/src/pipecat/audio/filters/krisp_filter.py index 7a54c5bd5..7d691f069 100644 --- a/src/pipecat/audio/filters/krisp_filter.py +++ b/src/pipecat/audio/filters/krisp_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -24,8 +24,7 @@ class KrispFilter(BaseAudioFilter): def __init__( self, sample_type: str = "PCM_16", channels: int = 1, model_path: str = None ) -> None: - """ - Initializes the KrispAudioProcessor with customizable audio processing settings. + """Initializes the KrispAudioProcessor with customizable audio processing settings. :param sample_type: The type of audio sample, default is 'PCM_16'. :param channels: Number of audio channels, default is 1. diff --git a/src/pipecat/audio/filters/noisereduce_filter.py b/src/pipecat/audio/filters/noisereduce_filter.py index ed68bb27a..c97a69c6f 100644 --- a/src/pipecat/audio/filters/noisereduce_filter.py +++ b/src/pipecat/audio/filters/noisereduce_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/mixers/base_audio_mixer.py b/src/pipecat/audio/mixers/base_audio_mixer.py index 0ba212d85..804948ed4 100644 --- a/src/pipecat/audio/mixers/base_audio_mixer.py +++ b/src/pipecat/audio/mixers/base_audio_mixer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/mixers/soundfile_mixer.py b/src/pipecat/audio/mixers/soundfile_mixer.py index 659ab0507..c49ef1184 100644 --- a/src/pipecat/audio/mixers/soundfile_mixer.py +++ b/src/pipecat/audio/mixers/soundfile_mixer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/utils.py b/src/pipecat/audio/utils.py index 5ef48dd1e..80785237f 100644 --- a/src/pipecat/audio/utils.py +++ b/src/pipecat/audio/utils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/vad/silero.py b/src/pipecat/audio/vad/silero.py index 28e0de716..28df15eba 100644 --- a/src/pipecat/audio/vad/silero.py +++ b/src/pipecat/audio/vad/silero.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/audio/vad/vad_analyzer.py b/src/pipecat/audio/vad/vad_analyzer.py index c291db7f3..6087a38b5 100644 --- a/src/pipecat/audio/vad/vad_analyzer.py +++ b/src/pipecat/audio/vad/vad_analyzer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/clocks/base_clock.py b/src/pipecat/clocks/base_clock.py index 79e17d5ba..e9ed3be80 100644 --- a/src/pipecat/clocks/base_clock.py +++ b/src/pipecat/clocks/base_clock.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/clocks/system_clock.py b/src/pipecat/clocks/system_clock.py index d919b6acd..4c434fc70 100644 --- a/src/pipecat/clocks/system_clock.py +++ b/src/pipecat/clocks/system_clock.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/frames/frames.proto b/src/pipecat/frames/frames.proto index 4c58d2a34..484042f18 100644 --- a/src/pipecat/frames/frames.proto +++ b/src/pipecat/frames/frames.proto @@ -1,5 +1,5 @@ // -// Copyright (c) 2024, Daily +// Copyright (c) 2025, Daily // // SPDX-License-Identifier: BSD 2-Clause License // diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index e9d942b4e..ac0075a59 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/pipeline/base_pipeline.py b/src/pipecat/pipeline/base_pipeline.py index a5ad68aa4..7debb3615 100644 --- a/src/pipecat/pipeline/base_pipeline.py +++ b/src/pipecat/pipeline/base_pipeline.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/pipeline/parallel_pipeline.py b/src/pipecat/pipeline/parallel_pipeline.py index b39258782..b277dc7dd 100644 --- a/src/pipecat/pipeline/parallel_pipeline.py +++ b/src/pipecat/pipeline/parallel_pipeline.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/pipeline/pipeline.py b/src/pipecat/pipeline/pipeline.py index 703f911fe..0aefd0bdf 100644 --- a/src/pipecat/pipeline/pipeline.py +++ b/src/pipecat/pipeline/pipeline.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/pipeline/runner.py b/src/pipecat/pipeline/runner.py index e83eab0f7..77d093c6a 100644 --- a/src/pipecat/pipeline/runner.py +++ b/src/pipecat/pipeline/runner.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/pipeline/sync_parallel_pipeline.py b/src/pipecat/pipeline/sync_parallel_pipeline.py index 5f9ff9ce7..2ba1e4695 100644 --- a/src/pipecat/pipeline/sync_parallel_pipeline.py +++ b/src/pipecat/pipeline/sync_parallel_pipeline.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 2ed5afcee..612ffb06a 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/gated.py b/src/pipecat/processors/aggregators/gated.py index 6976c9f10..975c92e9b 100644 --- a/src/pipecat/processors/aggregators/gated.py +++ b/src/pipecat/processors/aggregators/gated.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/gated_openai_llm_context.py b/src/pipecat/processors/aggregators/gated_openai_llm_context.py index 71a540dd4..7acda1015 100644 --- a/src/pipecat/processors/aggregators/gated_openai_llm_context.py +++ b/src/pipecat/processors/aggregators/gated_openai_llm_context.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/llm_response.py b/src/pipecat/processors/aggregators/llm_response.py index 479746471..349f7290c 100644 --- a/src/pipecat/processors/aggregators/llm_response.py +++ b/src/pipecat/processors/aggregators/llm_response.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/openai_llm_context.py b/src/pipecat/processors/aggregators/openai_llm_context.py index 853ac1baa..ec69407a5 100644 --- a/src/pipecat/processors/aggregators/openai_llm_context.py +++ b/src/pipecat/processors/aggregators/openai_llm_context.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/sentence.py b/src/pipecat/processors/aggregators/sentence.py index d0c593a83..3a97e4f66 100644 --- a/src/pipecat/processors/aggregators/sentence.py +++ b/src/pipecat/processors/aggregators/sentence.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/user_response.py b/src/pipecat/processors/aggregators/user_response.py index fd6b607a5..2eb44f070 100644 --- a/src/pipecat/processors/aggregators/user_response.py +++ b/src/pipecat/processors/aggregators/user_response.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/aggregators/vision_image_frame.py b/src/pipecat/processors/aggregators/vision_image_frame.py index d07337f06..cea51afba 100644 --- a/src/pipecat/processors/aggregators/vision_image_frame.py +++ b/src/pipecat/processors/aggregators/vision_image_frame.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/async_generator.py b/src/pipecat/processors/async_generator.py index 892f9d8b8..aad259dae 100644 --- a/src/pipecat/processors/async_generator.py +++ b/src/pipecat/processors/async_generator.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/audio/audio_buffer_processor.py b/src/pipecat/processors/audio/audio_buffer_processor.py index 488a251f0..5afc9ee43 100644 --- a/src/pipecat/processors/audio/audio_buffer_processor.py +++ b/src/pipecat/processors/audio/audio_buffer_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/audio/vad/silero.py b/src/pipecat/processors/audio/vad/silero.py index 2b115a8bb..62ebe267c 100644 --- a/src/pipecat/processors/audio/vad/silero.py +++ b/src/pipecat/processors/audio/vad/silero.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/frame_filter.py b/src/pipecat/processors/filters/frame_filter.py index 674670163..7c691ad11 100644 --- a/src/pipecat/processors/filters/frame_filter.py +++ b/src/pipecat/processors/filters/frame_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/function_filter.py b/src/pipecat/processors/filters/function_filter.py index 1d06be02c..860a2c78c 100644 --- a/src/pipecat/processors/filters/function_filter.py +++ b/src/pipecat/processors/filters/function_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/identity_filter.py b/src/pipecat/processors/filters/identity_filter.py index d6f896b73..78cba5186 100644 --- a/src/pipecat/processors/filters/identity_filter.py +++ b/src/pipecat/processors/filters/identity_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/null_filter.py b/src/pipecat/processors/filters/null_filter.py index 219cc149c..aae59a3c2 100644 --- a/src/pipecat/processors/filters/null_filter.py +++ b/src/pipecat/processors/filters/null_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/stt_mute_filter.py b/src/pipecat/processors/filters/stt_mute_filter.py index 914198e5a..bc709229a 100644 --- a/src/pipecat/processors/filters/stt_mute_filter.py +++ b/src/pipecat/processors/filters/stt_mute_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/filters/wake_check_filter.py b/src/pipecat/processors/filters/wake_check_filter.py index 860e45fa3..a9402c0c1 100644 --- a/src/pipecat/processors/filters/wake_check_filter.py +++ b/src/pipecat/processors/filters/wake_check_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -15,8 +15,7 @@ from pipecat.processors.frame_processor import FrameDirection, FrameProcessor class WakeCheckFilter(FrameProcessor): - """ - This filter looks for wake phrases in the transcription frames and only passes through frames + """This filter looks for wake phrases in the transcription frames and only passes through frames after a wake phrase has been detected. It also has a keepalive timeout to allow for a brief period of continued conversation after a wake phrase has been detected. """ diff --git a/src/pipecat/processors/filters/wake_notifier_filter.py b/src/pipecat/processors/filters/wake_notifier_filter.py index a7f074ccb..e0e0cce3a 100644 --- a/src/pipecat/processors/filters/wake_notifier_filter.py +++ b/src/pipecat/processors/filters/wake_notifier_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/frame_processor.py b/src/pipecat/processors/frame_processor.py index e3dfe0bce..ad15de62e 100644 --- a/src/pipecat/processors/frame_processor.py +++ b/src/pipecat/processors/frame_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/frameworks/langchain.py b/src/pipecat/processors/frameworks/langchain.py index 47789cec9..d1a691b2f 100644 --- a/src/pipecat/processors/frameworks/langchain.py +++ b/src/pipecat/processors/frameworks/langchain.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/frameworks/rtvi.py b/src/pipecat/processors/frameworks/rtvi.py index 471bdbb88..92ff01df4 100644 --- a/src/pipecat/processors/frameworks/rtvi.py +++ b/src/pipecat/processors/frameworks/rtvi.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/gstreamer/pipeline_source.py b/src/pipecat/processors/gstreamer/pipeline_source.py index 7d2c23c69..9de5ff08e 100644 --- a/src/pipecat/processors/gstreamer/pipeline_source.py +++ b/src/pipecat/processors/gstreamer/pipeline_source.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/idle_frame_processor.py b/src/pipecat/processors/idle_frame_processor.py index d1c86e3ab..b8b447f94 100644 --- a/src/pipecat/processors/idle_frame_processor.py +++ b/src/pipecat/processors/idle_frame_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/logger.py b/src/pipecat/processors/logger.py index 8c925af2a..d4f2615a2 100644 --- a/src/pipecat/processors/logger.py +++ b/src/pipecat/processors/logger.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/metrics/frame_processor_metrics.py b/src/pipecat/processors/metrics/frame_processor_metrics.py index 9cacb9a7f..8d1ba4191 100644 --- a/src/pipecat/processors/metrics/frame_processor_metrics.py +++ b/src/pipecat/processors/metrics/frame_processor_metrics.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/metrics/sentry.py b/src/pipecat/processors/metrics/sentry.py index bc93170ff..5a092dc98 100644 --- a/src/pipecat/processors/metrics/sentry.py +++ b/src/pipecat/processors/metrics/sentry.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/text_transformer.py b/src/pipecat/processors/text_transformer.py index 79e9b885e..5e84551ed 100644 --- a/src/pipecat/processors/text_transformer.py +++ b/src/pipecat/processors/text_transformer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/transcript_processor.py b/src/pipecat/processors/transcript_processor.py index 1e5e97c59..884d4988f 100644 --- a/src/pipecat/processors/transcript_processor.py +++ b/src/pipecat/processors/transcript_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/processors/user_idle_processor.py b/src/pipecat/processors/user_idle_processor.py index 160c49908..e5bbbdd1c 100644 --- a/src/pipecat/processors/user_idle_processor.py +++ b/src/pipecat/processors/user_idle_processor.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/serializers/base_serializer.py b/src/pipecat/serializers/base_serializer.py index 00f2f2a68..c421ca46c 100644 --- a/src/pipecat/serializers/base_serializer.py +++ b/src/pipecat/serializers/base_serializer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/serializers/livekit.py b/src/pipecat/serializers/livekit.py index aafc00b23..50f341e5e 100644 --- a/src/pipecat/serializers/livekit.py +++ b/src/pipecat/serializers/livekit.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/serializers/protobuf.py b/src/pipecat/serializers/protobuf.py index 4e6ade772..c2179afa2 100644 --- a/src/pipecat/serializers/protobuf.py +++ b/src/pipecat/serializers/protobuf.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -58,7 +58,9 @@ class ProtobufFrameSerializer(FrameSerializer): return proto_frame.SerializeToString() def deserialize(self, data: str | bytes) -> Frame | None: - """Returns a Frame object from a Frame protobuf. Used to convert frames + """Returns a Frame object from a Frame protobuf. + + Used to convert frames passed over the wire as protobufs to Frame objects used in pipelines and frame processors. @@ -75,7 +77,6 @@ class ProtobufFrameSerializer(FrameSerializer): ... text="Hello there!", participantId="123", timestamp="2021-01-01"))) TranscriptionFrame(text='Hello there!', participantId='123', timestamp='2021-01-01') """ - proto = frame_protos.Frame.FromString(data) which = proto.WhichOneof("frame") if which not in self.DESERIALIZABLE_FIELDS: diff --git a/src/pipecat/serializers/twilio.py b/src/pipecat/serializers/twilio.py index 3c3648365..577664dde 100644 --- a/src/pipecat/serializers/twilio.py +++ b/src/pipecat/serializers/twilio.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index e0f16e220..45b1c4639 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/anthropic.py b/src/pipecat/services/anthropic.py index 93cff6f9e..807333c17 100644 --- a/src/pipecat/services/anthropic.py +++ b/src/pipecat/services/anthropic.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/assemblyai.py b/src/pipecat/services/assemblyai.py index 36a7e92ff..f17cfa903 100644 --- a/src/pipecat/services/assemblyai.py +++ b/src/pipecat/services/assemblyai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/aws.py b/src/pipecat/services/aws.py index cd09e2ff8..b6a763f16 100644 --- a/src/pipecat/services/aws.py +++ b/src/pipecat/services/aws.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/azure.py b/src/pipecat/services/azure.py index a95ff7d3c..def2ffccb 100644 --- a/src/pipecat/services/azure.py +++ b/src/pipecat/services/azure.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/canonical.py b/src/pipecat/services/canonical.py index 376168c3f..d2671b250 100644 --- a/src/pipecat/services/canonical.py +++ b/src/pipecat/services/canonical.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -43,7 +43,6 @@ class CanonicalMetricsService(AIService): uploads it to Canonical Voice API for audio processing. Args: - call_id (str): Your unique identifier for the call. This is used to match the call in the Canonical Voice system to the call in your system. assistant (str): Identifier for the AI assistant. This can be whatever you want, it's intended for you convenience so you can distinguish between different assistants and a grouping mechanism for calls. diff --git a/src/pipecat/services/cartesia.py b/src/pipecat/services/cartesia.py index f4d5935a8..540d7cfd8 100644 --- a/src/pipecat/services/cartesia.py +++ b/src/pipecat/services/cartesia.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/cerebras.py b/src/pipecat/services/cerebras.py index a0cc81803..7867d5fae 100644 --- a/src/pipecat/services/cerebras.py +++ b/src/pipecat/services/cerebras.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/deepgram.py b/src/pipecat/services/deepgram.py index 8d3def5be..132656385 100644 --- a/src/pipecat/services/deepgram.py +++ b/src/pipecat/services/deepgram.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/elevenlabs.py b/src/pipecat/services/elevenlabs.py index 65135642d..988eecfab 100644 --- a/src/pipecat/services/elevenlabs.py +++ b/src/pipecat/services/elevenlabs.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/fal.py b/src/pipecat/services/fal.py index a7b2b7e30..e949a5713 100644 --- a/src/pipecat/services/fal.py +++ b/src/pipecat/services/fal.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/fireworks.py b/src/pipecat/services/fireworks.py index 1f7d22b36..0f43bf377 100644 --- a/src/pipecat/services/fireworks.py +++ b/src/pipecat/services/fireworks.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/fish.py b/src/pipecat/services/fish.py index cc1302fa9..91e19163b 100644 --- a/src/pipecat/services/fish.py +++ b/src/pipecat/services/fish.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py b/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py index 953137106..95642b19d 100644 --- a/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py +++ b/src/pipecat/services/gemini_multimodal_live/audio_transcriber.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/gemini_multimodal_live/events.py b/src/pipecat/services/gemini_multimodal_live/events.py index edb2867f9..0d5bc802f 100644 --- a/src/pipecat/services/gemini_multimodal_live/events.py +++ b/src/pipecat/services/gemini_multimodal_live/events.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/gemini_multimodal_live/gemini.py b/src/pipecat/services/gemini_multimodal_live/gemini.py index 1fded9e1a..dd4375486 100644 --- a/src/pipecat/services/gemini_multimodal_live/gemini.py +++ b/src/pipecat/services/gemini_multimodal_live/gemini.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/gladia.py b/src/pipecat/services/gladia.py index 8909c4bb2..33f8b0cf1 100644 --- a/src/pipecat/services/gladia.py +++ b/src/pipecat/services/gladia.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/google.py b/src/pipecat/services/google.py index d724d2776..b52c1f6dc 100644 --- a/src/pipecat/services/google.py +++ b/src/pipecat/services/google.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/grok.py b/src/pipecat/services/grok.py index a6a1b3a64..f38d86966 100644 --- a/src/pipecat/services/grok.py +++ b/src/pipecat/services/grok.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/groq.py b/src/pipecat/services/groq.py index c9b49d636..f035f7607 100644 --- a/src/pipecat/services/groq.py +++ b/src/pipecat/services/groq.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/lmnt.py b/src/pipecat/services/lmnt.py index 5393e6653..ba931dc38 100644 --- a/src/pipecat/services/lmnt.py +++ b/src/pipecat/services/lmnt.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/moondream.py b/src/pipecat/services/moondream.py index 400028700..eebc12ce6 100644 --- a/src/pipecat/services/moondream.py +++ b/src/pipecat/services/moondream.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -23,9 +23,7 @@ except ModuleNotFoundError as e: def detect_device(): - """ - Detects the appropriate device to run on, and return the device and dtype. - """ + """Detects the appropriate device to run on, and return the device and dtype.""" try: import intel_extension_for_pytorch diff --git a/src/pipecat/services/nim.py b/src/pipecat/services/nim.py index 2b57a5047..3250ba420 100644 --- a/src/pipecat/services/nim.py +++ b/src/pipecat/services/nim.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/ollama.py b/src/pipecat/services/ollama.py index 0a6a4ce6a..1e74b303e 100644 --- a/src/pipecat/services/ollama.py +++ b/src/pipecat/services/ollama.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai.py b/src/pipecat/services/openai.py index 159db8d2f..f614ec575 100644 --- a/src/pipecat/services/openai.py +++ b/src/pipecat/services/openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai_realtime_beta/context.py b/src/pipecat/services/openai_realtime_beta/context.py index 9ea8dd691..049195188 100644 --- a/src/pipecat/services/openai_realtime_beta/context.py +++ b/src/pipecat/services/openai_realtime_beta/context.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 0515012e3..f757f8f74 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai_realtime_beta/frames.py b/src/pipecat/services/openai_realtime_beta/frames.py index 54bdcd467..22fa7e87c 100644 --- a/src/pipecat/services/openai_realtime_beta/frames.py +++ b/src/pipecat/services/openai_realtime_beta/frames.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openai_realtime_beta/openai.py b/src/pipecat/services/openai_realtime_beta/openai.py index 2e952bea1..ea5d4c41e 100644 --- a/src/pipecat/services/openai_realtime_beta/openai.py +++ b/src/pipecat/services/openai_realtime_beta/openai.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/openpipe.py b/src/pipecat/services/openpipe.py index fb6362b15..ea36c9129 100644 --- a/src/pipecat/services/openpipe.py +++ b/src/pipecat/services/openpipe.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/playht.py b/src/pipecat/services/playht.py index 115198c02..ddbcd76b1 100644 --- a/src/pipecat/services/playht.py +++ b/src/pipecat/services/playht.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 79614cb76..4bfa56b20 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/riva.py b/src/pipecat/services/riva.py index 8ab0e99d8..77396f84a 100644 --- a/src/pipecat/services/riva.py +++ b/src/pipecat/services/riva.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/simli.py b/src/pipecat/services/simli.py index 1f88838be..19825daf3 100644 --- a/src/pipecat/services/simli.py +++ b/src/pipecat/services/simli.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/tavus.py b/src/pipecat/services/tavus.py index b701af00d..df41a9fc9 100644 --- a/src/pipecat/services/tavus.py +++ b/src/pipecat/services/tavus.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/together.py b/src/pipecat/services/together.py index e18e9650d..e43c7a104 100644 --- a/src/pipecat/services/together.py +++ b/src/pipecat/services/together.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/services/whisper.py b/src/pipecat/services/whisper.py index 266ff2b84..6cbcf4793 100644 --- a/src/pipecat/services/whisper.py +++ b/src/pipecat/services/whisper.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -61,7 +61,8 @@ class WhisperSTTService(SegmentedSTTService): def _load(self): """Loads the Whisper model. Note that if this is the first time - this model is being run, it will take time to download.""" + this model is being run, it will take time to download. + """ logger.debug("Loading Whisper model...") self._model = WhisperModel( self.model_name, device=self._device, compute_type=self._compute_type diff --git a/src/pipecat/services/xtts.py b/src/pipecat/services/xtts.py index 900701ffa..47b91967e 100644 --- a/src/pipecat/services/xtts.py +++ b/src/pipecat/services/xtts.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/sync/base_notifier.py b/src/pipecat/sync/base_notifier.py index c7770ab26..757c1326b 100644 --- a/src/pipecat/sync/base_notifier.py +++ b/src/pipecat/sync/base_notifier.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/sync/event_notifier.py b/src/pipecat/sync/event_notifier.py index f02dcbdae..ac87419da 100644 --- a/src/pipecat/sync/event_notifier.py +++ b/src/pipecat/sync/event_notifier.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transcriptions/language.py b/src/pipecat/transcriptions/language.py index ee16c14a7..0bba6b2f1 100644 --- a/src/pipecat/transcriptions/language.py +++ b/src/pipecat/transcriptions/language.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 025a5bed2..165e9ca4b 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/base_output.py b/src/pipecat/transports/base_output.py index c787405cf..8f2715b5c 100644 --- a/src/pipecat/transports/base_output.py +++ b/src/pipecat/transports/base_output.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/base_transport.py b/src/pipecat/transports/base_transport.py index 27f6cab75..8d997ab08 100644 --- a/src/pipecat/transports/base_transport.py +++ b/src/pipecat/transports/base_transport.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/local/audio.py b/src/pipecat/transports/local/audio.py index d62c6bd48..4e03701d8 100644 --- a/src/pipecat/transports/local/audio.py +++ b/src/pipecat/transports/local/audio.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/local/tk.py b/src/pipecat/transports/local/tk.py index e720f3907..98ad908c7 100644 --- a/src/pipecat/transports/local/tk.py +++ b/src/pipecat/transports/local/tk.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/network/fastapi_websocket.py b/src/pipecat/transports/network/fastapi_websocket.py index e7ac8b5a5..02270d5c0 100644 --- a/src/pipecat/transports/network/fastapi_websocket.py +++ b/src/pipecat/transports/network/fastapi_websocket.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/network/websocket_server.py b/src/pipecat/transports/network/websocket_server.py index 58d104038..4bbfeef7e 100644 --- a/src/pipecat/transports/network/websocket_server.py +++ b/src/pipecat/transports/network/websocket_server.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index c00352d98..ae43f0efa 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/services/helpers/daily_rest.py b/src/pipecat/transports/services/helpers/daily_rest.py index f2b7c7e59..c17af897c 100644 --- a/src/pipecat/transports/services/helpers/daily_rest.py +++ b/src/pipecat/transports/services/helpers/daily_rest.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/transports/services/livekit.py b/src/pipecat/transports/services/livekit.py index 81a0ffdd1..0cf1d16dc 100644 --- a/src/pipecat/transports/services/livekit.py +++ b/src/pipecat/transports/services/livekit.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 936764345..9c4cab12c 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/utils/text/base_text_filter.py b/src/pipecat/utils/text/base_text_filter.py index 4e814d7f1..bcc370874 100644 --- a/src/pipecat/utils/text/base_text_filter.py +++ b/src/pipecat/utils/text/base_text_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/utils/text/markdown_text_filter.py b/src/pipecat/utils/text/markdown_text_filter.py index 2344d3779..6ec705d69 100644 --- a/src/pipecat/utils/text/markdown_text_filter.py +++ b/src/pipecat/utils/text/markdown_text_filter.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -117,8 +117,7 @@ class MarkdownTextFilter(BaseTextFilter): # def _remove_code_blocks(self, text: str) -> str: - """ - Main method to remove code blocks from the input text. + """Main method to remove code blocks from the input text. Handles interruptions and delegates to specific methods based on the current state. """ if self._interrupted: @@ -135,8 +134,7 @@ class MarkdownTextFilter(BaseTextFilter): return self._handle_not_in_code_block(match, text, code_block_pattern) def _handle_in_code_block(self, match, text): - """ - Handle text when we're currently inside a code block. + """Handle text when we're currently inside a code block. If we find the end of the block, return text after it. Otherwise, skip the content. """ if match: @@ -146,8 +144,7 @@ class MarkdownTextFilter(BaseTextFilter): return "" # Skip content inside code block def _handle_not_in_code_block(self, match, text, code_block_pattern): - """ - Handle text when we're not currently inside a code block. + """Handle text when we're not currently inside a code block. Delegate to specific methods based on whether we find a code block delimiter. """ if not match: @@ -159,16 +156,14 @@ class MarkdownTextFilter(BaseTextFilter): return self._handle_code_block_within_text(text, code_block_pattern) def _handle_start_of_code_block(self, text, start_index): - """ - Handle the case where we find the start of a code block. + """Handle the case where we find the start of a code block. Return any text before the code block and set the state to inside a code block. """ self._in_code_block = True return text[:start_index].strip() def _handle_code_block_within_text(self, text, code_block_pattern): - """ - Handle the case where we find a code block within the text. + """Handle the case where we find a code block within the text. If it's a complete code block, remove it and return surrounding text. If it's the start of a code block, return text before it and set state. """ @@ -182,8 +177,7 @@ class MarkdownTextFilter(BaseTextFilter): # Filter tables # def remove_tables(self, text: str) -> str: - """ - Remove tables from the input text, handling cases where + """Remove tables from the input text, handling cases where both start and end tags are in the same input. """ if self._interrupted: diff --git a/src/pipecat/utils/time.py b/src/pipecat/utils/time.py index 0f6ca1076..154d1f0a7 100644 --- a/src/pipecat/utils/time.py +++ b/src/pipecat/utils/time.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/utils/utils.py b/src/pipecat/utils/utils.py index 14f1b541a..aa9f1d971 100644 --- a/src/pipecat/utils/utils.py +++ b/src/pipecat/utils/utils.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # @@ -11,8 +11,7 @@ _ID = itertools.count() def obj_id() -> int: - """ - Generate a unique id for an object. + """Generate a unique id for an object. >>> obj_id() 0 diff --git a/src/pipecat/vad/silero.py b/src/pipecat/vad/silero.py index 7ec938dbd..e78826dfe 100644 --- a/src/pipecat/vad/silero.py +++ b/src/pipecat/vad/silero.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/src/pipecat/vad/vad_analyzer.py b/src/pipecat/vad/vad_analyzer.py index b29b10ef9..14cb5d059 100644 --- a/src/pipecat/vad/vad_analyzer.py +++ b/src/pipecat/vad/vad_analyzer.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # diff --git a/tests/test_langchain.py b/tests/test_langchain.py index bed3f907a..97c97f133 100644 --- a/tests/test_langchain.py +++ b/tests/test_langchain.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2024, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License #