Improve TOC in sidebar, fix missing services

.github/workflows/generate_docs.yaml

@@ -1,47 +0,0 @@
-name: Generate API Documentation
-
-on:
-  release:
-    types: [published] # Run on new release
-  workflow_dispatch: # Manual trigger
-
-jobs:
-  update-docs:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-      pull-requests: write
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.12'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r docs/api/requirements.txt
-          pip install .
-
-      - name: Generate API documentation
-        run: |
-          cd docs/api
-          python generate_docs.py
-
-      - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v5
-        with:
-          commit-message: 'docs: Update API documentation'
-          title: 'docs: Update API documentation'
-          body: |
-            Automated PR to update API documentation.
-
-            - Generated using `docs/api/generate_docs.py`
-            - Triggered by: ${{ github.event_name }}
-          branch: update-api-docs
-          delete-branch: true
-          labels: |
-            documentation

docs/api/conf.py

@@ -1,5 +1,12 @@
+import importlib
+import logging
 import sys
 from pathlib import Path
+from typing import Dict, List, Optional
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger("sphinx-build")
 
 # Add source directory to path
 docs_dir = Path(__file__).parent
@@ -17,6 +24,7 @@ extensions = [
     "sphinx.ext.napoleon",
     "sphinx.ext.viewcode",
     "sphinx.ext.intersphinx",
+    "sphinx.ext.coverage",
 ]
 
 # Napoleon settings
@@ -32,13 +40,72 @@ autodoc_default_options = {
     "undoc-members": True,
     "exclude-members": "__weakref__",
     "no-index": True,
+    "show-inheritance": True,
 }
 
 # HTML output settings
 html_theme = "sphinx_rtd_theme"
 html_static_path = ["_static"]
 autodoc_typehints = "description"
-html_show_sphinx = False  # Remove "Built with Sphinx"
+html_show_sphinx = False
+
+
+def get_installed_services() -> Dict[str, Optional[str]]:
+    """Scan for installed pipecat services and return their status.
+    Returns a dictionary of service names and their import status/error message.
+    """
+    services_dir = project_root / "src" / "pipecat" / "services"
+    services_status = {}
+
+    if not services_dir.exists():
+        logger.warning(f"Services directory not found: {services_dir}")
+        return services_status
+
+    for item in services_dir.iterdir():
+        if item.is_dir() and not item.name.startswith("_") and not item.name == "to_be_updated":
+            service_name = item.name
+            try:
+                module = importlib.import_module(f"pipecat.services.{service_name}")
+                services_status[service_name] = None  # None indicates success
+                logger.info(f"Found service: {service_name} at {module.__file__}")
+            except ImportError as e:
+                services_status[service_name] = str(e)
+                logger.warning(f"Failed to import {service_name}: {e}")
+
+    return services_status
+
+
+def generate_services_rst() -> str:
+    """Generate RST content for services section."""
+    services = get_installed_services()
+
+    # Sort services into successful and failed imports
+    successful = [name for name, status in services.items() if status is None]
+    failed = [(name, status) for name, status in services.items() if status is not None]
+
+    rst_content = [
+        "Services",
+        "~~~~~~~~",
+        "",
+        "Successfully Detected Services:",
+        "",
+    ]
+
+    for service in sorted(successful):
+        rst_content.append(f"* :mod:`pipecat.services.{service}`")
+
+    if failed:
+        rst_content.extend(
+            [
+                "",
+                "Services with Import Issues:",
+                "",
+            ]
+        )
+        for service, error in sorted(failed):
+            rst_content.append(f"* {service} (Import failed: {error})")
+
+    return "\n".join(rst_content)
 
 
 def setup(app):
@@ -55,12 +122,21 @@ def setup(app):
         import shutil
 
         shutil.rmtree(output_dir)
+        logger.info(f"Cleaned existing documentation in {output_dir}")
 
-    print(f"Generating API documentation...")
+    logger.info(f"Generating API documentation...")
-    print(f"Output directory: {output_dir}")
+    logger.info(f"Output directory: {output_dir}")
-    print(f"Source directory: {source_dir}")
+    logger.info(f"Source directory: {source_dir}")
+
+    # Get installed services
+    services = get_installed_services()
+    logger.info(f"Found {len(services)} services")
+    for service, status in services.items():
+        if status is None:
+            logger.info(f"Service available: {service}")
+        else:
+            logger.warning(f"Service import failed: {service} - {status}")
 
-    # Similar exclusions as in your generate_docs.py
     excludes = [
         str(project_root / "src/pipecat/processors/gstreamer"),
         str(project_root / "src/pipecat/transports/network"),
@@ -72,7 +148,27 @@ def setup(app):
     ]
 
     try:
-        main(["-f", "-e", "-M", "--no-toc", "-o", output_dir, source_dir] + excludes)
+        main(
-        print("API documentation generated successfully!")
+            [
+                "-f",
+                "-e",
+                "-M",
+                "--no-toc",
+                "--separate",
+                "--module-first",
+                "-o",
+                output_dir,
+                source_dir,
+            ]
+            + excludes
+        )
+
+        logger.info("API documentation generated successfully!")
+
+        # Generate services index file
+        services_index = Path(output_dir) / "services_index.rst"
+        services_index.write_text(generate_services_rst())
+        logger.info(f"Generated services index at {services_index}")
+
     except Exception as e:
-        print(f"Error generating API documentation: {e}")
+        logger.error(f"Error generating API documentation: {e}", exc_info=True)

docs/api/generate_docs.py

@@ -1,104 +0,0 @@
-#!/usr/bin/env python3
-
-import shutil
-import subprocess
-from pathlib import Path
-
-
-def run_command(command: list[str]) -> None:
-    """Run a command and exit if it fails."""
-    print(f"Running: {' '.join(command)}")
-    try:
-        subprocess.run(command, check=True)
-    except subprocess.CalledProcessError as e:
-        print(f"Warning: Command failed: {' '.join(command)}")
-        print(f"Error: {e}")
-
-
-def main():
-    docs_dir = Path(__file__).parent
-    project_root = docs_dir.parent.parent
-
-    # Install documentation requirements
-    requirements_file = docs_dir / "requirements.txt"
-    run_command(["pip", "install", "-r", str(requirements_file)])
-
-    # Install from project root, not docs directory
-    run_command(["pip", "install", "-e", str(project_root)])
-
-    # Install all service dependencies
-    services = [
-        "anthropic",
-        "assemblyai",
-        "aws",
-        "azure",
-        "canonical",
-        "cartesia",
-        # "daily",
-        "deepgram",
-        "elevenlabs",
-        "fal",
-        "fireworks",
-        "gladia",
-        "google",
-        "grok",
-        "groq",
-        "langchain",
-        # "livekit",
-        "lmnt",
-        "moondream",
-        "nim",
-        "noisereduce",
-        "openai",
-        "openpipe",
-        "playht",
-        "silero",
-        "soundfile",
-        "websocket",
-        "whisper",
-    ]
-
-    extras = ",".join(services)
-    try:
-        run_command(["pip", "install", "-e", f"{str(project_root)}[{extras}]"])
-    except Exception as e:
-        print(f"Warning: Some dependencies failed to install: {e}")
-
-    # Clean old files
-    api_dir = docs_dir / "api"
-    build_dir = docs_dir / "_build"
-    for dir in [api_dir, build_dir]:
-        if dir.exists():
-            shutil.rmtree(dir)
-
-    # Generate API documentation
-    run_command(
-        [
-            "sphinx-apidoc",
-            "-f",  # Force overwrite
-            "-e",  # Put each module on its own page
-            "-M",  # Put module documentation before submodule
-            "--no-toc",  # Don't generate modules.rst (cleaner structure)
-            "-o",
-            str(api_dir),  # Output directory
-            str(project_root / "src/pipecat"),
-            # Exclude problematic files and directories
-            str(project_root / "src/pipecat/processors/gstreamer"),  # Optional gstreamer
-            str(project_root / "src/pipecat/transports/network"),  # Pydantic issues
-            str(project_root / "src/pipecat/transports/services"),  # Pydantic issues
-            str(project_root / "src/pipecat/transports/local"),  # Optional dependencies
-            str(project_root / "src/pipecat/services/to_be_updated"),  # Exclude to_be_updated
-            "**/test_*.py",  # Test files
-            "**/tests/*.py",  # Test files
-        ]
-    )
-
-    # Build HTML documentation
-    run_command(["sphinx-build", "-b", "html", str(docs_dir), str(build_dir / "html")])
-
-    print("\nDocumentation generated successfully!")
-    print(f"HTML docs: {build_dir}/html/index.html")
-
-
-if __name__ == "__main__":
-    main()

docs/api/index.rst

@@ -13,61 +13,55 @@ Quick Links
 * `GitHub Repository <https://github.com/pipecat-ai/pipecat>`_
 * `Website <https://pipecat.ai>`_
 
-
 API Reference
 -------------
 
 Core Components
 ~~~~~~~~~~~~~~~
 
-* :mod:`pipecat.frames`
+* :mod:`Frames <pipecat.frames>`
-* :mod:`pipecat.processors`
+* :mod:`Processors <pipecat.processors>`
-* :mod:`pipecat.pipeline`
+* :mod:`Pipeline <pipecat.pipeline>`
 
 Audio Processing
 ~~~~~~~~~~~~~~~~
 
-* :mod:`pipecat.audio`
+* :mod:`Audio <pipecat.audio>`
-* :mod:`pipecat.vad`
+* :mod:`VAD <pipecat.vad>`
-
-Services
-~~~~~~~~
-
-* :mod:`pipecat.services`
 
 Transport & Serialization
 ~~~~~~~~~~~~~~~~~~~~~~~~~
 
-* :mod:`pipecat.transports`
+* :mod:`Transports <pipecat.transports>`
-* :mod:`pipecat.serializers`
+* :mod:`Serializers <pipecat.serializers>`
 
 Utilities
 ~~~~~~~~~
 
-* :mod:`pipecat.clocks`
+* :mod:`Clocks <pipecat.clocks>`
-* :mod:`pipecat.metrics`
+* :mod:`Metrics <pipecat.metrics>`
-* :mod:`pipecat.sync`
+* :mod:`Sync <pipecat.sync>`
-* :mod:`pipecat.transcriptions`
+* :mod:`Transcriptions <pipecat.transcriptions>`
-* :mod:`pipecat.utils`
+* :mod:`Utils <pipecat.utils>`
 
 .. toctree::
    :maxdepth: 2
    :caption: API Reference
    :hidden:
 
-   api/pipecat.audio
+   Audio <api/pipecat.audio>
-   api/pipecat.clocks
+   Clocks <api/pipecat.clocks>
-   api/pipecat.frames
+   Frames <api/pipecat.frames>
-   api/pipecat.metrics
+   Metrics <api/pipecat.metrics>
-   api/pipecat.pipeline
+   Pipeline <api/pipecat.pipeline>
-   api/pipecat.processors
+   Processors <api/pipecat.processors>
-   api/pipecat.serializers
+   Serializers <api/pipecat.serializers>
-   api/pipecat.services
+   Services <api/pipecat.services>
-   api/pipecat.sync
+   Sync <api/pipecat.sync>
-   api/pipecat.transcriptions
+   Transcriptions <api/pipecat.transcriptions>
-   api/pipecat.transports
+   Transports <api/pipecat.transports>
-   api/pipecat.utils
+   Utils <api/pipecat.utils>
-   api/pipecat.vad
+   VAD <api/pipecat.vad>
 
 Indices and tables
 ==================

docs/api/requirements.txt

@@ -3,4 +3,4 @@ sphinx-rtd-theme
 sphinx-markdown-builder
 sphinx-autodoc-typehints
 toml
-pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,silero,soundfile,websocket,whisper]
+pipecat-ai[anthropic,assemblyai,aws,azure,canonical,cartesia,deepgram,elevenlabs,fal,fireworks,gladia,google,grok,groq,krisp,langchain,lmnt,moondream,nim,noisereduce,openai,openpipe,playht,riva,silero,simli,soundfile,websocket,whisper]

src/pipecat/services/assemblyai.py

@@ -1,3 +1,9 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
 import asyncio
 from typing import AsyncGenerator
 
@@ -67,8 +73,7 @@ class AssemblyAISTTService(STTService):
         await self._disconnect()
 
     async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:
-        """
+        """Process an audio chunk for STT transcription.
-        Process an audio chunk for STT transcription.
 
         This method streams the audio data to AssemblyAI for real-time transcription.
         Transcription results are handled asynchronously via callback functions.
@@ -83,8 +88,7 @@ class AssemblyAISTTService(STTService):
         yield None
 
     async def _connect(self):
-        """
+        """Establish a connection to the AssemblyAI real-time transcription service.
-        Establish a connection to the AssemblyAI real-time transcription service.
 
         This method sets up the necessary callback functions and initializes the
         AssemblyAI transcriber.
@@ -95,8 +99,7 @@ class AssemblyAISTTService(STTService):
             logger.info(f"{self}: Connected to AssemblyAI")
 
         def on_data(transcript: aai.RealtimeTranscript):
-            """
+            """Callback for handling incoming transcription data.
-            Callback for handling incoming transcription data.
 
             This function runs in a separate thread from the main asyncio event loop.
             It creates appropriate transcription frames and schedules them to be
@@ -121,8 +124,7 @@ class AssemblyAISTTService(STTService):
             asyncio.run_coroutine_threadsafe(self.push_frame(frame), self._loop)
 
         def on_error(error: aai.RealtimeError):
-            """
+            """Callback for handling errors from AssemblyAI.
-            Callback for handling errors from AssemblyAI.
 
             Like on_data, this runs in a separate thread and schedules error
             handling in the main event loop.

src/pipecat/services/rime.py

@@ -1,3 +1,9 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
 from typing import AsyncGenerator, Optional
 
 import aiohttp