Add support for OpenTelemetry tracing (#1729)

* Also added TurnTrackingObserver, TurnTraceObserver, foundational 29, open-telemetry-example
2025-05-13 17:18:11 -04:00
parent 6f4d94f91b
commit 50f6235edb
52 changed files with 2797 additions and 31 deletions
--- a/examples/foundational/29-turn-tracking-observer.py
+++ b/examples/foundational/29-turn-tracking-observer.py
@@ -0,0 +1,119 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+
+load_dotenv(override=True)
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace):
+    logger.info(f"Starting bot")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(),
+        ),
+    )
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+        },
+    ]
+
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    pipeline = Pipeline(
+        [
+            transport.input(),  # Transport user input
+            stt,
+            context_aggregator.user(),  # User responses
+            llm,  # LLM
+            tts,  # TTS
+            transport.output(),  # Transport bot output
+            context_aggregator.assistant(),  # Assistant spoken responses
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            allow_interruptions=True,
+            enable_metrics=True,
+            enable_usage_metrics=True,
+            report_only_initial_ttfb=True,
+        ),
+    )
+
+    turn_observer = task.turn_tracking_observer
+    if turn_observer:
+
+        @turn_observer.event_handler("on_turn_started")
+        async def on_turn_started(observer, turn_number):
+            logger.info(f"🔄 Turn {turn_number} started")
+
+        @turn_observer.event_handler("on_turn_ended")
+        async def on_turn_ended(observer, turn_number, duration, was_interrupted):
+            if was_interrupted:
+                logger.info(f"🔄 Turn {turn_number} interrupted after {duration:.2f}s")
+            else:
+                logger.info(f"🏁 Turn {turn_number} completed in {duration:.2f}s")
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Client connected")
+        # Kick off the conversation.
+        messages.append({"role": "system", "content": "Please introduce yourself to the user."})
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Client disconnected")
+
+    @transport.event_handler("on_client_closed")
+    async def on_client_closed(transport, client):
+        logger.info(f"Client closed connection")
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=False)
+
+    await runner.run(task)
+
+
+if __name__ == "__main__":
+    from run import main
+
+    main()
--- a/examples/open-telemetry-tracing/README.md
+++ b/examples/open-telemetry-tracing/README.md
@@ -0,0 +1,169 @@
+# OpenTelemetry Tracing for Pipecat
+
+This demo showcases OpenTelemetry tracing integration for Pipecat services, allowing you to visualize service calls, performance metrics, and dependencies in a Jaeger dashboard.
+
+## Features
+
+- **Hierarchical Tracing**: Track entire conversations, turns, and service calls
+- **Service Tracing**: Detailed spans for TTS, STT, and LLM services with rich context
+- **TTFB Metrics**: Capture Time To First Byte metrics for latency analysis
+- **Usage Statistics**: Track character counts for TTS and token usage for LLMs
+- **Flexible Exporters**: Use Jaeger, Zipkin, or any OpenTelemetry-compatible backend
+
+## Trace Structure
+
+Traces are organized hierarchically:
+
+```
+Conversation (conversation-uuid)
+├── turn-1
+│   ├── stt_deepgramsttservice
+│   ├── llm_openaillmservice
+│   └── tts_cartesiattsservice
+└── turn-2
+    ├── stt_deepgramsttservice
+    ├── llm_openaillmservice
+    └── tts_cartesiattsservice
+    turn-N
+    └── ...
+```
+
+This organization helps you track conversation-to-conversation and turn-to-turn.
+
+## Setup Instructions
+
+### 1. Start the Jaeger Container
+
+Run Jaeger in Docker to collect and visualize traces:
+
+```bash
+docker run -d --name jaeger \
+  -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
+  -p 16686:16686 \
+  -p 4317:4317 \
+  -p 4318:4318 \
+  jaegertracing/all-in-one:latest
+```
+
+### 2. Environment Configuration
+
+Create a `.env` file with your API keys and enable tracing:
+
+```
+ENABLE_TRACING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317  # Point to your preferred backend
+# OTEL_CONSOLE_EXPORT=true  # Set to any value for debug output to console
+
+# Service API keys
+DEEPGRAM_API_KEY=your_key_here
+CARTESIA_API_KEY=your_key_here
+OPENAI_API_KEY=your_key_here
+```
+
+### 3. Configure Your Pipeline Task
+
+Enable tracing in your Pipecat application:
+
+```python
+# Initialize OpenTelemetry with your chosen exporter
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+exporter = OTLPSpanExporter(
+    endpoint="http://localhost:4317",  # Jaeger OTLP endpoint
+    insecure=True,
+)
+
+setup_tracing(
+    service_name="pipecat-demo",
+    exporter=exporter,
+    console_export=os.getenv("OTEL_CONSOLE_EXPORT", "false").lower() == "true",
+)
+
+# Enable tracing in your PipelineTask
+task = PipelineTask(
+    pipeline,
+    params=PipelineParams(
+        allow_interruptions=True,
+        enable_metrics=True,  # Required for some service metrics
+    ),
+    enable_tracing=True,  # Enables both turn and conversation tracing
+    conversation_id="customer-123",  # Optional - will auto-generate if not provided
+)
+```
+
+### 4. Exporter Options
+
+While this demo uses Jaeger, you can configure any OpenTelemetry-compatible exporter:
+
+#### Jaeger (Default for the demo)
+
+```python
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+exporter = OTLPSpanExporter(
+    endpoint="http://localhost:4317",  # Jaeger OTLP endpoint
+    insecure=True,
+)
+```
+
+#### Cloud Providers
+
+Many cloud providers offer OpenTelemetry-compatible observability services:
+
+- AWS X-Ray
+- Google Cloud Trace
+- Azure Monitor
+- Datadog APM
+
+See the OpenTelemetry documentation for specific exporter configurations:
+https://opentelemetry.io/ecosystem/vendors/
+
+### 5. Install Dependencies
+
+```bash
+pip install -r requirements.txt
+```
+
+### 6. Run the Demo
+
+```bash
+python bot.py
+```
+
+### 7. View Traces in Jaeger
+
+Open your browser to [http://localhost:16686](http://localhost:16686) and select the "pipecat-demo" service to view traces.
+
+## Understanding the Traces
+
+- **Conversation Spans**: The top-level span representing an entire conversation
+- **Turn Spans**: Child spans of conversations that represent each turn in the dialog
+- **Service Spans**: Detailed service operations nested under turns
+- **Service Attributes**: Each service includes rich context about its operation:
+  - **TTS**: Voice ID, character count, service type
+  - **STT**: Transcription text, language, model
+  - **LLM**: Messages, tokens used, model, service configuration
+- **Metrics**: Performance data like `metrics.ttfb_ms` and processing durations
+
+## How It Works
+
+The tracing system consists of:
+
+1. **TurnTrackingObserver**: Detects conversation turns
+2. **TurnTraceObserver**: Creates spans for turns and conversations
+3. **Service Decorators**: `@traced_tts`, `@traced_stt`, `@traced_llm` for service-specific tracing
+4. **Context Providers**: Share context between different parts of the pipeline
+
+## Troubleshooting
+
+- **No Traces in Jaeger**: Ensure the Docker container is running and the OTLP endpoint is correct
+- **Debugging Traces**: Set `OTEL_CONSOLE_EXPORT=true` to print traces to the console for debugging
+- **Missing Metrics**: Check that `enable_metrics=True` in PipelineParams
+- **Connection Errors**: Verify network connectivity to the Jaeger container
+- **Exporter Issues**: Try the Console exporter (`OTEL_CONSOLE_EXPORT=true`) to verify tracing works
+- **Other Backends**: If using a different backend, ensure you've configured the correct exporter and endpoint
+
+## References
+
+- [OpenTelemetry Python Documentation](https://opentelemetry-python.readthedocs.io/)
+- [Jaeger Documentation](https://www.jaegertracing.io/docs/latest/)
--- a/examples/open-telemetry-tracing/bot.py
+++ b/examples/open-telemetry-tracing/bot.py
@@ -0,0 +1,159 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import os
+
+from dotenv import load_dotenv
+from loguru import logger
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+
+from pipecat.adapters.schemas.function_schema import FunctionSchema
+from pipecat.adapters.schemas.tools_schema import ToolsSchema
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.frames.frames import TTSSpeakFrame
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.llm_service import FunctionCallParams
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import TransportParams
+from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
+from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
+from pipecat.utils.tracing.setup import setup_tracing
+
+load_dotenv(override=True)
+
+IS_TRACING_ENABLED = bool(os.getenv("ENABLE_TRACING"))
+
+# Initialize tracing if enabled
+if IS_TRACING_ENABLED:
+    # Create the exporter
+    otlp_exporter = OTLPSpanExporter(
+        endpoint=os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4317"),
+        insecure=True,
+    )
+
+    # Set up tracing with the exporter
+    setup_tracing(
+        service_name="pipecat-demo",
+        exporter=otlp_exporter,
+        console_export=bool(os.getenv("OTEL_CONSOLE_EXPORT")),
+    )
+    logger.info("OpenTelemetry tracing initialized")
+
+
+async def fetch_weather_from_api(params: FunctionCallParams):
+    await params.llm.push_frame(TTSSpeakFrame("Let me check on that."))
+    await params.result_callback({"conditions": "nice", "temperature": "75"})
+
+
+async def run_bot(webrtc_connection: SmallWebRTCConnection, _: argparse.Namespace):
+    logger.info(f"Starting bot")
+
+    transport = SmallWebRTCTransport(
+        webrtc_connection=webrtc_connection,
+        params=TransportParams(
+            audio_in_enabled=True,
+            audio_out_enabled=True,
+            vad_analyzer=SileroVADAnalyzer(),
+        ),
+    )
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(
+        api_key=os.getenv("OPENAI_API_KEY"), params=OpenAILLMService.InputParams(temperature=0.5)
+    )
+
+    # You can also register a function_name of None to get all functions
+    # sent to the same callback with an additional function_name parameter.
+    llm.register_function("get_current_weather", fetch_weather_from_api)
+
+    weather_function = FunctionSchema(
+        name="get_current_weather",
+        description="Get the current weather",
+        properties={
+            "location": {
+                "type": "string",
+                "description": "The city and state, e.g. San Francisco, CA",
+            },
+            "format": {
+                "type": "string",
+                "enum": ["celsius", "fahrenheit"],
+                "description": "The temperature unit to use. Infer this from the user's location.",
+            },
+        },
+        required=["location", "format"],
+    )
+    tools = ToolsSchema(standard_tools=[weather_function])
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+        },
+    ]
+
+    context = OpenAILLMContext(messages, tools)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    pipeline = Pipeline(
+        [
+            transport.input(),
+            stt,
+            context_aggregator.user(),
+            llm,
+            tts,
+            transport.output(),
+            context_aggregator.assistant(),
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            allow_interruptions=True,
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+        enable_tracing=IS_TRACING_ENABLED,
+        # Optionally, add a conversation ID to track the conversation
+        # conversation_id="8df26cc1-6db0-4a7a-9930-1e037c8f1fa2",
+    )
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Client connected")
+        # Kick off the conversation.
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Client disconnected")
+
+    @transport.event_handler("on_client_closed")
+    async def on_client_closed(transport, client):
+        logger.info(f"Client closed connection")
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=False)
+
+    await runner.run(task)
+
+
+if __name__ == "__main__":
+    from run import main
+
+    main()
--- a/examples/open-telemetry-tracing/env.example
+++ b/examples/open-telemetry-tracing/env.example
@@ -0,0 +1,10 @@
+DEEPGRAM_API_KEY=your_deepgram_key
+CARTESIA_API_KEY=your_cartesia_key
+OPENAI_API_KEY=your_openai_key
+
+# Set to any value to enable tracing
+ENABLE_TRACING=true
+# OTLP endpoint (defaults to localhost:4317 if not set)
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+# Set to any value to enable console output for debugging
+# OTEL_CONSOLE_EXPORT=true
--- a/examples/open-telemetry-tracing/requirements.txt
+++ b/examples/open-telemetry-tracing/requirements.txt
@@ -0,0 +1,6 @@
+fastapi
+uvicorn
+python-dotenv
+pipecat-ai[webrtc,silero,cartesia,deepgram,openai,tracing]
+pipecat-ai-small-webrtc-prebuilt
+opentelemetry-exporter-otlp-proto-grpc
--- a/examples/open-telemetry-tracing/run.py
+++ b/examples/open-telemetry-tracing/run.py
@@ -0,0 +1,205 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import asyncio
+import importlib.util
+import os
+import sys
+from contextlib import asynccontextmanager
+from inspect import iscoroutinefunction, signature
+from typing import Any, Callable, Dict, Optional, Tuple
+
+import uvicorn
+from dotenv import load_dotenv
+from fastapi import BackgroundTasks, FastAPI
+from fastapi.responses import RedirectResponse
+from loguru import logger
+from pipecat_ai_small_webrtc_prebuilt.frontend import SmallWebRTCPrebuiltUI
+
+from pipecat.transports.network.webrtc_connection import IceServer, SmallWebRTCConnection
+
+# Load environment variables
+load_dotenv(override=True)
+
+app = FastAPI()
+
+# Store connections by pc_id
+pcs_map: Dict[str, SmallWebRTCConnection] = {}
+
+ice_servers = [
+    IceServer(
+        urls="stun:stun.l.google.com:19302",
+    )
+]
+
+# Mount the frontend at /
+app.mount("/client", SmallWebRTCPrebuiltUI)
+
+# Store program arguments
+args: argparse.Namespace = argparse.Namespace()
+
+# Store the bot module and function info
+bot_module: Any = None
+run_bot_func: Optional[Callable] = None
+is_webrtc_bot: bool = True
+
+
+def import_bot_file(file_path: str) -> Tuple[Any, Callable, bool]:
+    """Dynamically import the bot file and determine how to run it.
+
+    Returns:
+        tuple: (module, run_function, is_webrtc_bot)
+          - module: The imported module
+          - run_function: Either run_bot or main function
+          - is_webrtc_bot: True if run_bot function exists and accepts a WebRTC connection
+    """
+    if not os.path.exists(file_path):
+        raise FileNotFoundError(f"Bot file not found: {file_path}")
+
+    # Extract module name without extension
+    module_name = os.path.splitext(os.path.basename(file_path))[0]
+
+    # Load the module
+    spec = importlib.util.spec_from_file_location(module_name, file_path)
+    if not spec or not spec.loader:
+        raise ImportError(f"Could not load spec for {file_path}")
+
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[module_name] = module
+    spec.loader.exec_module(module)
+
+    # Check for run_bot function first
+    if hasattr(module, "run_bot"):
+        run_func = module.run_bot
+        # Check if the function accepts a WebRTC connection
+        sig = signature(run_func)
+        is_webrtc = len(sig.parameters) > 0
+        return module, run_func, is_webrtc
+
+    # Fall back to main function
+    if hasattr(module, "main") and iscoroutinefunction(module.main):
+        return module, module.main, False
+
+    raise AttributeError(f"No run_bot or async main function found in {file_path}")
+
+
+@app.get("/", include_in_schema=False)
+async def root_redirect():
+    return RedirectResponse(url="/client/")
+
+
+@app.post("/api/offer")
+async def offer(request: dict, background_tasks: BackgroundTasks):
+    global run_bot_func, is_webrtc_bot
+
+    if not run_bot_func:
+        raise RuntimeError("No bot file has been loaded")
+
+    if not is_webrtc_bot:
+        return {
+            "error": "This bot doesn't support WebRTC connections, it's running in standalone mode"
+        }
+
+    pc_id = request.get("pc_id")
+
+    if pc_id and pc_id in pcs_map:
+        pipecat_connection = pcs_map[pc_id]
+        logger.info(f"Reusing existing connection for pc_id: {pc_id}")
+        await pipecat_connection.renegotiate(
+            sdp=request["sdp"], type=request["type"], restart_pc=request.get("restart_pc", False)
+        )
+    else:
+        pipecat_connection = SmallWebRTCConnection(ice_servers)
+        await pipecat_connection.initialize(sdp=request["sdp"], type=request["type"])
+
+        @pipecat_connection.event_handler("closed")
+        async def handle_disconnected(webrtc_connection: SmallWebRTCConnection):
+            logger.info(f"Discarding peer connection for pc_id: {webrtc_connection.pc_id}")
+            pcs_map.pop(webrtc_connection.pc_id, None)
+
+        # We've already checked that run_bot_func exists
+        assert run_bot_func is not None
+        background_tasks.add_task(run_bot_func, pipecat_connection, args)
+
+    answer = pipecat_connection.get_answer()
+    # Updating the peer connection inside the map
+    pcs_map[answer["pc_id"]] = pipecat_connection
+
+    return answer
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    yield  # Run app
+    coros = [pc.close() for pc in pcs_map.values()]
+    await asyncio.gather(*coros)
+    pcs_map.clear()
+
+
+async def run_standalone_bot() -> None:
+    """Run a standalone bot that doesn't require WebRTC"""
+    global run_bot_func
+    if run_bot_func is not None:
+        await run_bot_func()
+    else:
+        raise RuntimeError("No bot function available to run")
+
+
+def main(parser: Optional[argparse.ArgumentParser] = None):
+    global args
+
+    if not parser:
+        parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
+    parser.add_argument("bot_file", nargs="?", help="Path to the bot file", default=None)
+    parser.add_argument(
+        "--host", default="localhost", help="Host for HTTP server (default: localhost)"
+    )
+    parser.add_argument(
+        "--port", type=int, default=7860, help="Port for HTTP server (default: 7860)"
+    )
+    parser.add_argument("--verbose", "-v", action="count", default=0)
+    args = parser.parse_args()
+
+    logger.remove(0)
+    if args.verbose:
+        logger.add(sys.stderr, level="TRACE")
+    else:
+        logger.add(sys.stderr, level="DEBUG")
+
+    # Infer the bot file from the caller if not provided explicitly
+    bot_file = args.bot_file
+    if bot_file is None:
+        # Get the __file__ of the script that called main()
+        import inspect
+
+        caller_frame = inspect.stack()[1]
+        caller_globals = caller_frame.frame.f_globals
+        bot_file = caller_globals.get("__file__")
+
+    if not bot_file:
+        print("❌ Could not determine the bot file. Pass it explicitly to main().")
+        sys.exit(1)
+
+    # Import the bot file
+    try:
+        global run_bot_func, bot_module, is_webrtc_bot
+        bot_module, run_bot_func, is_webrtc_bot = import_bot_file(bot_file)
+        logger.info(f"Successfully loaded bot from {bot_file}")
+
+        if is_webrtc_bot:
+            logger.info("Detected WebRTC-compatible bot, starting web server...")
+            uvicorn.run(app, host=args.host, port=args.port)
+        else:
+            logger.info("Detected standalone bot, running directly...")
+            asyncio.run(run_standalone_bot())
+    except Exception as e:
+        logger.error(f"Error loading bot file: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()