From 414dcf9810a91e163b679cd5996cc1d818549293 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 12 Dec 2024 11:06:09 -0500 Subject: [PATCH] Improve TOC in sidebar, fix missing services --- .github/workflows/generate_docs.yaml | 47 ----------- docs/api/conf.py | 112 +++++++++++++++++++++++++-- docs/api/generate_docs.py | 104 ------------------------- docs/api/index.rst | 56 ++++++-------- docs/api/requirements.txt | 2 +- src/pipecat/services/assemblyai.py | 18 +++-- src/pipecat/services/rime.py | 6 ++ 7 files changed, 146 insertions(+), 199 deletions(-) delete mode 100644 .github/workflows/generate_docs.yaml delete mode 100644 docs/api/generate_docs.py diff --git a/.github/workflows/generate_docs.yaml b/.github/workflows/generate_docs.yaml deleted file mode 100644 index 242984d8a..000000000 --- a/.github/workflows/generate_docs.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Generate API Documentation - -on: - release: - types: [published] # Run on new release - workflow_dispatch: # Manual trigger - -jobs: - update-docs: - runs-on: ubuntu-latest - permissions: - contents: write - pull-requests: write - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r docs/api/requirements.txt - pip install . - - - name: Generate API documentation - run: | - cd docs/api - python generate_docs.py - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v5 - with: - commit-message: 'docs: Update API documentation' - title: 'docs: Update API documentation' - body: | - Automated PR to update API documentation. - - - Generated using `docs/api/generate_docs.py` - - Triggered by: ${{ github.event_name }} - branch: update-api-docs - delete-branch: true - labels: | - documentation diff --git a/docs/api/conf.py b/docs/api/conf.py index fffaf90b2..c046092e2 100644 --- a/docs/api/conf.py +++ b/docs/api/conf.py @@ -1,5 +1,12 @@ +import importlib +import logging import sys from pathlib import Path +from typing import Dict, List, Optional + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") +logger = logging.getLogger("sphinx-build") # Add source directory to path docs_dir = Path(__file__).parent @@ -17,6 +24,7 @@ extensions = [ "sphinx.ext.napoleon", "sphinx.ext.viewcode", "sphinx.ext.intersphinx", + "sphinx.ext.coverage", ] # Napoleon settings @@ -32,13 +40,72 @@ autodoc_default_options = { "undoc-members": True, "exclude-members": "__weakref__", "no-index": True, + "show-inheritance": True, } # HTML output settings html_theme = "sphinx_rtd_theme" html_static_path = ["_static"] autodoc_typehints = "description" -html_show_sphinx = False # Remove "Built with Sphinx" +html_show_sphinx = False + + +def get_installed_services() -> Dict[str, Optional[str]]: + """Scan for installed pipecat services and return their status. + Returns a dictionary of service names and their import status/error message. + """ + services_dir = project_root / "src" / "pipecat" / "services" + services_status = {} + + if not services_dir.exists(): + logger.warning(f"Services directory not found: {services_dir}") + return services_status + + for item in services_dir.iterdir(): + if item.is_dir() and not item.name.startswith("_") and not item.name == "to_be_updated": + service_name = item.name + try: + module = importlib.import_module(f"pipecat.services.{service_name}") + services_status[service_name] = None # None indicates success + logger.info(f"Found service: {service_name} at {module.__file__}") + except ImportError as e: + services_status[service_name] = str(e) + logger.warning(f"Failed to import {service_name}: {e}") + + return services_status + + +def generate_services_rst() -> str: + """Generate RST content for services section.""" + services = get_installed_services() + + # Sort services into successful and failed imports + successful = [name for name, status in services.items() if status is None] + failed = [(name, status) for name, status in services.items() if status is not None] + + rst_content = [ + "Services", + "~~~~~~~~", + "", + "Successfully Detected Services:", + "", + ] + + for service in sorted(successful): + rst_content.append(f"* :mod:`pipecat.services.{service}`") + + if failed: + rst_content.extend( + [ + "", + "Services with Import Issues:", + "", + ] + ) + for service, error in sorted(failed): + rst_content.append(f"* {service} (Import failed: {error})") + + return "\n".join(rst_content) def setup(app): @@ -55,12 +122,21 @@ def setup(app): import shutil shutil.rmtree(output_dir) + logger.info(f"Cleaned existing documentation in {output_dir}") - print(f"Generating API documentation...") - print(f"Output directory: {output_dir}") - print(f"Source directory: {source_dir}") + logger.info(f"Generating API documentation...") + logger.info(f"Output directory: {output_dir}") + logger.info(f"Source directory: {source_dir}") + + # Get installed services + services = get_installed_services() + logger.info(f"Found {len(services)} services") + for service, status in services.items(): + if status is None: + logger.info(f"Service available: {service}") + else: + logger.warning(f"Service import failed: {service} - {status}") - # Similar exclusions as in your generate_docs.py excludes = [ str(project_root / "src/pipecat/processors/gstreamer"), str(project_root / "src/pipecat/transports/network"), @@ -72,7 +148,27 @@ def setup(app): ] try: - main(["-f", "-e", "-M", "--no-toc", "-o", output_dir, source_dir] + excludes) - print("API documentation generated successfully!") + main( + [ + "-f", + "-e", + "-M", + "--no-toc", + "--separate", + "--module-first", + "-o", + output_dir, + source_dir, + ] + + excludes + ) + + logger.info("API documentation generated successfully!") + + # Generate services index file + services_index = Path(output_dir) / "services_index.rst" + services_index.write_text(generate_services_rst()) + logger.info(f"Generated services index at {services_index}") + except Exception as e: - print(f"Error generating API documentation: {e}") + logger.error(f"Error generating API documentation: {e}", exc_info=True) diff --git a/docs/api/generate_docs.py b/docs/api/generate_docs.py deleted file mode 100644 index 972ea3c89..000000000 --- a/docs/api/generate_docs.py +++ /dev/null @@ -1,104 +0,0 @@ -#!/usr/bin/env python3 - -import shutil -import subprocess -from pathlib import Path - - -def run_command(command: list[str]) -> None: - """Run a command and exit if it fails.""" - print(f"Running: {' '.join(command)}") - try: - subprocess.run(command, check=True) - except subprocess.CalledProcessError as e: - print(f"Warning: Command failed: {' '.join(command)}") - print(f"Error: {e}") - - -def main(): - docs_dir = Path(__file__).parent - project_root = docs_dir.parent.parent - - # Install documentation requirements - requirements_file = docs_dir / "requirements.txt" - run_command(["pip", "install", "-r", str(requirements_file)]) - - # Install from project root, not docs directory - run_command(["pip", "install", "-e", str(project_root)]) - - # Install all service dependencies - services = [ - "anthropic", - "assemblyai", - "aws", - "azure", - "canonical", - "cartesia", - # "daily", - "deepgram", - "elevenlabs", - "fal", - "fireworks", - "gladia", - "google", - "grok", - "groq", - "langchain", - # "livekit", - "lmnt", - "moondream", - "nim", - "noisereduce", - "openai", - "openpipe", - "playht", - "silero", - "soundfile", - "websocket", - "whisper", - ] - - extras = ",".join(services) - try: - run_command(["pip", "install", "-e", f"{str(project_root)}[{extras}]"]) - except Exception as e: - print(f"Warning: Some dependencies failed to install: {e}") - - # Clean old files - api_dir = docs_dir / "api" - build_dir = docs_dir / "_build" - for dir in [api_dir, build_dir]: - if dir.exists(): - shutil.rmtree(dir) - - # Generate API documentation - run_command( - [ - "sphinx-apidoc", - "-f", # Force overwrite - "-e", # Put each module on its own page - "-M", # Put module documentation before submodule - "--no-toc", # Don't generate modules.rst (cleaner structure) - "-o", - str(api_dir), # Output directory - str(project_root / "src/pipecat"), - # Exclude problematic files and directories - str(project_root / "src/pipecat/processors/gstreamer"), # Optional gstreamer - str(project_root / "src/pipecat/transports/network"), # Pydantic issues - str(project_root / "src/pipecat/transports/services"), # Pydantic issues - str(project_root / "src/pipecat/transports/local"), # Optional dependencies - str(project_root / "src/pipecat/services/to_be_updated"), # Exclude to_be_updated - "**/test_*.py", # Test files - "**/tests/*.py", # Test files - ] - ) - - # Build HTML documentation - run_command(["sphinx-build", "-b", "html", str(docs_dir), str(build_dir / "html")]) - - print("\nDocumentation generated successfully!") - print(f"HTML docs: {build_dir}/html/index.html") - - -if __name__ == "__main__": - main() diff --git a/docs/api/index.rst b/docs/api/index.rst index 68afcb980..a878c0c10 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -13,61 +13,55 @@ Quick Links * `GitHub Repository `_ * `Website `_ - API Reference ------------- Core Components ~~~~~~~~~~~~~~~ -* :mod:`pipecat.frames` -* :mod:`pipecat.processors` -* :mod:`pipecat.pipeline` +* :mod:`Frames ` +* :mod:`Processors ` +* :mod:`Pipeline ` Audio Processing ~~~~~~~~~~~~~~~~ -* :mod:`pipecat.audio` -* :mod:`pipecat.vad` - -Services -~~~~~~~~ - -* :mod:`pipecat.services` +* :mod:`Audio ` +* :mod:`VAD ` Transport & Serialization ~~~~~~~~~~~~~~~~~~~~~~~~~ -* :mod:`pipecat.transports` -* :mod:`pipecat.serializers` +* :mod:`Transports ` +* :mod:`Serializers ` Utilities ~~~~~~~~~ -* :mod:`pipecat.clocks` -* :mod:`pipecat.metrics` -* :mod:`pipecat.sync` -* :mod:`pipecat.transcriptions` -* :mod:`pipecat.utils` +* :mod:`Clocks ` +* :mod:`Metrics ` +* :mod:`Sync ` +* :mod:`Transcriptions ` +* :mod:`Utils ` .. toctree:: :maxdepth: 2 :caption: API Reference :hidden: - api/pipecat.audio - api/pipecat.clocks - api/pipecat.frames - api/pipecat.metrics - api/pipecat.pipeline - api/pipecat.processors - api/pipecat.serializers - api/pipecat.services - api/pipecat.sync - api/pipecat.transcriptions - api/pipecat.transports - api/pipecat.utils - api/pipecat.vad + Audio + Clocks + Frames + Metrics + Pipeline + Processors + Serializers + Services + Sync + Transcriptions + Transports + Utils + VAD Indices and tables ================== diff --git a/docs/api/requirements.txt b/docs/api/requirements.txt index f571b6ce0..d57bcd00f 100644 --- a/docs/api/requirements.txt +++ b/docs/api/requirements.txt @@ -3,4 +3,4 @@ sphinx-rtd-theme sphinx-markdown-builder sphinx-autodoc-typehints toml -pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,silero,soundfile,websocket,whisper] +pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,riva,silero,simli,soundfile,websocket,whisper] \ No newline at end of file diff --git a/src/pipecat/services/assemblyai.py b/src/pipecat/services/assemblyai.py index a96589e5a..36a7e92ff 100644 --- a/src/pipecat/services/assemblyai.py +++ b/src/pipecat/services/assemblyai.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + import asyncio from typing import AsyncGenerator @@ -67,8 +73,7 @@ class AssemblyAISTTService(STTService): await self._disconnect() async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: - """ - Process an audio chunk for STT transcription. + """Process an audio chunk for STT transcription. This method streams the audio data to AssemblyAI for real-time transcription. Transcription results are handled asynchronously via callback functions. @@ -83,8 +88,7 @@ class AssemblyAISTTService(STTService): yield None async def _connect(self): - """ - Establish a connection to the AssemblyAI real-time transcription service. + """Establish a connection to the AssemblyAI real-time transcription service. This method sets up the necessary callback functions and initializes the AssemblyAI transcriber. @@ -95,8 +99,7 @@ class AssemblyAISTTService(STTService): logger.info(f"{self}: Connected to AssemblyAI") def on_data(transcript: aai.RealtimeTranscript): - """ - Callback for handling incoming transcription data. + """Callback for handling incoming transcription data. This function runs in a separate thread from the main asyncio event loop. It creates appropriate transcription frames and schedules them to be @@ -121,8 +124,7 @@ class AssemblyAISTTService(STTService): asyncio.run_coroutine_threadsafe(self.push_frame(frame), self._loop) def on_error(error: aai.RealtimeError): - """ - Callback for handling errors from AssemblyAI. + """Callback for handling errors from AssemblyAI. Like on_data, this runs in a separate thread and schedules error handling in the main event loop. diff --git a/src/pipecat/services/rime.py b/src/pipecat/services/rime.py index 6cf2c8129..79614cb76 100644 --- a/src/pipecat/services/rime.py +++ b/src/pipecat/services/rime.py @@ -1,3 +1,9 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + from typing import AsyncGenerator, Optional import aiohttp