Fal Smart Turn example

examples/fal-smart-turn/server/Dockerfile

@@ -0,0 +1,8 @@
+FROM dailyco/pipecat-base:latest
+
+COPY ./requirements.txt requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+COPY ./assets assets
+COPY ./bot.py bot.py

examples/fal-smart-turn/server/bot.py

@@ -0,0 +1,301 @@
+#
+# Copyright (c) 2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import asyncio
+import os
+import sys
+
+import aiohttp
+from dotenv import load_dotenv
+from loguru import logger
+from PIL import Image
+from pipecatcloud.agent import DailySessionArguments
+
+from pipecat.audio.turn.smart_turn.fal_smart_turn import FalSmartTurnAnalyzer
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    Frame,
+    MetricsFrame,
+    OutputImageRawFrame,
+    SpriteFrame,
+)
+from pipecat.metrics.metrics import SmartTurnMetricsData
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.processors.frameworks.rtvi import (
+    RTVIConfig,
+    RTVIObserver,
+    RTVIProcessor,
+    RTVIServerMessageFrame,
+)
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.google.llm import GoogleLLMService
+from pipecat.transports.services.daily import DailyParams, DailyTransport
+
+load_dotenv(override=True)
+
+# Check if we're in local development mode
+LOCAL = os.getenv("LOCAL")
+
+logger.remove()
+logger.add(sys.stderr, level="DEBUG")
+
+sprites = []
+script_dir = os.path.dirname(__file__)
+
+# Load sequential animation frames
+for i in range(1, 26):
+    # Build the full path to the image file
+    full_path = os.path.join(script_dir, f"assets/robot0{i}.png")
+    # Get the filename without the extension to use as the dictionary key
+    # Open the image and convert it to bytes
+    with Image.open(full_path) as img:
+        sprites.append(OutputImageRawFrame(image=img.tobytes(), size=img.size, format=img.format))
+
+# Create a smooth animation by adding reversed frames
+flipped = sprites[::-1]
+sprites.extend(flipped)
+
+# Define static and animated states
+quiet_frame = sprites[0]  # Static frame for when bot is listening
+talking_frame = SpriteFrame(images=sprites)  # Animation sequence for when bot is talking
+
+
+class TalkingAnimation(FrameProcessor):
+    """Manages the bot's visual animation states.
+
+    Switches between static (listening) and animated (talking) states based on
+    the bot's current speaking status.
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._is_talking = False
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and update animation state.
+
+        Args:
+            frame: The incoming frame to process
+            direction: The direction of frame flow in the pipeline
+        """
+        await super().process_frame(frame, direction)
+
+        # Switch to talking animation when bot starts speaking
+        if isinstance(frame, BotStartedSpeakingFrame):
+            if not self._is_talking:
+                await self.push_frame(talking_frame)
+                self._is_talking = True
+        # Return to static frame when bot stops speaking
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            await self.push_frame(quiet_frame)
+            self._is_talking = False
+
+        await self.push_frame(frame, direction)
+
+
+class SmartTurnMetricsProcessor(FrameProcessor):
+    """Processes the metrics data from Smart Turn Analyzer.
+
+    This processor is responsible for handling smart turn metrics data
+    and forwarding it to the client UI via RTVI.
+    """
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        """Process incoming frames and handle Smart Turn metrics.
+
+        Args:
+            frame: The incoming frame to process
+            direction: The direction of frame flow in the pipeline
+        """
+        await super().process_frame(frame, direction)
+
+        # Handle Smart Turn metrics
+        if isinstance(frame, MetricsFrame):
+            for metrics in frame.data:
+                if isinstance(metrics, SmartTurnMetricsData):
+                    logger.info(f"Smart Turn metrics: {metrics}")
+
+                    # Create a payload with the smart turn prediction data
+                    smart_turn_data = {
+                        "type": "smart_turn_result",
+                        "is_complete": metrics.is_complete,
+                        "probability": metrics.probability,
+                        "inference_time_ms": metrics.inference_time_ms,
+                        "server_total_time_ms": metrics.server_total_time_ms,
+                        "e2e_processing_time_ms": metrics.e2e_processing_time_ms,
+                    }
+
+                    # Send the data to the client via RTVI
+                    rtvi_frame = RTVIServerMessageFrame(data=smart_turn_data)
+                    await self.push_frame(rtvi_frame)
+
+        await self.push_frame(frame, direction)
+
+
+async def main(transport: DailyTransport):
+    # Configure your STT, LLM, and TTS services here
+    # Swap out different processors or properties to customize your bot
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+    llm = GoogleLLMService(api_key=os.getenv("GOOGLE_API_KEY"))
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    # Set up the initial context for the conversation
+    # You can specified initial system and assistant messages here
+    messages = [
+        {
+            "role": "system",
+            "content": "You are Chatbot, a friendly, helpful robot. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way, but keep your responses brief. Start by introducing yourself.",
+        },
+    ]
+
+    # This sets up the LLM context by providing messages and tools
+    context = OpenAILLMContext(messages)
+    context_aggregator = llm.create_context_aggregator(context)
+
+    ta = TalkingAnimation()
+    smart_turn_metrics_processor = SmartTurnMetricsProcessor()
+
+    # RTVI events for Pipecat client UI
+    rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
+
+    # A core voice AI pipeline
+    # Add additional processors to customize the bot's behavior
+    pipeline = Pipeline(
+        [
+            transport.input(),
+            rtvi,
+            smart_turn_metrics_processor,
+            stt,
+            context_aggregator.user(),
+            llm,
+            tts,
+            ta,
+            transport.output(),
+            context_aggregator.assistant(),
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            allow_interruptions=True,
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+        observers=[RTVIObserver(rtvi)],
+    )
+
+    @rtvi.event_handler("on_client_ready")
+    async def on_client_ready(rtvi):
+        logger.debug("Client ready event received")
+        await rtvi.set_bot_ready()
+        # Kick off the conversation
+        await task.queue_frames([context_aggregator.user().get_context_frame()])
+
+    @transport.event_handler("on_first_participant_joined")
+    async def on_first_participant_joined(transport, participant):
+        logger.info("First participant joined: {}", participant["id"])
+        # Push a static frame to show the bot is listening
+        await task.queue_frame(quiet_frame)
+
+    @transport.event_handler("on_participant_left")
+    async def on_participant_left(transport, participant, reason):
+        logger.info("Participant left: {}", participant)
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=False, force_gc=True)
+
+    await runner.run(task)
+
+
+async def bot(args: DailySessionArguments):
+    """Main bot entry point compatible with the FastAPI route handler.
+
+    Args:
+        room_url: The Daily room URL
+        token: The Daily room token
+        body: The configuration object from the request body
+        session_id: The session ID for logging
+    """
+    from pipecat.audio.filters.krisp_filter import KrispFilter
+
+    logger.info(f"Bot process initialized {args.room_url} {args.token}")
+    async with aiohttp.ClientSession() as session:
+        transport = DailyTransport(
+            args.room_url,
+            args.token,
+            "Word Wrangler Bot",
+            DailyParams(
+                audio_in_filter=KrispFilter(),
+                audio_out_enabled=True,
+                camera_out_enabled=True,
+                camera_out_width=1024,
+                camera_out_height=576,
+                vad_enabled=True,
+                vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+                vad_audio_passthrough=True,
+                turn_analyzer=FalSmartTurnAnalyzer(
+                    api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=session
+                ),
+            ),
+        )
+
+        try:
+            await main(transport)
+            logger.info("Bot process completed")
+        except Exception as e:
+            logger.exception(f"Error in bot process: {str(e)}")
+            raise
+
+
+# Local development
+async def local_daily():
+    """Daily transport for local development."""
+    from runner import configure
+
+    try:
+        async with aiohttp.ClientSession() as session:
+            (room_url, token) = await configure(session)
+            transport = DailyTransport(
+                room_url,
+                token,
+                bot_name="Bot",
+                params=DailyParams(
+                    audio_out_enabled=True,
+                    camera_out_enabled=True,
+                    camera_out_width=1024,
+                    camera_out_height=576,
+                    vad_enabled=True,
+                    vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+                    vad_audio_passthrough=True,
+                    turn_analyzer=FalSmartTurnAnalyzer(
+                        api_key=os.getenv("FAL_SMART_TURN_API_KEY"), aiohttp_session=session
+                    ),
+                ),
+            )
+
+            await main(transport)
+    except Exception as e:
+        logger.exception(f"Error in local development mode: {e}")
+
+
+# Local development entry point
+if LOCAL and __name__ == "__main__":
+    try:
+        asyncio.run(local_daily())
+    except Exception as e:
+        logger.exception(f"Failed to run in local mode: {e}")

examples/fal-smart-turn/server/build.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+VERSION="0.1"
+DOCKER_USERNAME=""
+AGENT_NAME="pcc-smart-turn"
+
+# Build the Docker image with the correct context
+echo "Building Docker image..."
+docker build --platform=linux/arm64 -t "$DOCKER_USERNAME/$AGENT_NAME:$VERSION" -t "$DOCKER_USERNAME/$AGENT_NAME:latest" .
+
+# Push the Docker images
+echo "Pushing Docker image $DOCKER_USERNAME/$AGENT_NAME:$VERSION..."
+docker push "$DOCKER_USERNAME/$AGENT_NAME:$VERSION"
+
+echo "Pushing Docker image $DOCKER_USERNAME/$AGENT_NAME:latest..."
+docker push "$DOCKER_USERNAME/$AGENT_NAME:latest"
+
+echo "Successfully built and pushed $DOCKER_USERNAME/$AGENT_NAME:$VERSION and $DOCKER_USERNAME/$AGENT_NAME:latest"

examples/fal-smart-turn/server/env.example

@@ -0,0 +1,4 @@
+GOOGLE_API_KEY=
+CARTESIA_API_KEY=
+DEEPGRAM_API_KEY=
+FAL_SMART_TURN_API_KEY=

examples/fal-smart-turn/server/pcc-deploy.toml

@@ -0,0 +1,7 @@
+agent_name = "pcc-smart-turn"
+image = "your-username/pcc-smart-turn:0.1"
+secret_set = "pcc-smart-turn-secrets"
+enable_krisp = true
+
+[scaling]
+	min_instances = 0

examples/fal-smart-turn/server/requirements.txt

@@ -0,0 +1,3 @@
+pipecatcloud
+pipecat-ai[google,daily,deepgram,cartesia,silero]
+python-dotenv

examples/fal-smart-turn/server/runner.py

@@ -0,0 +1,56 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import argparse
+import os
+
+import aiohttp
+
+from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper
+
+
+async def configure(aiohttp_session: aiohttp.ClientSession):
+    """Configure the Daily room and Daily REST helper."""
+    parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
+    parser.add_argument(
+        "-u", "--url", type=str, required=False, help="URL of the Daily room to join"
+    )
+    parser.add_argument(
+        "-k",
+        "--apikey",
+        type=str,
+        required=False,
+        help="Daily API Key (needed to create an owner token for the room)",
+    )
+
+    args, unknown = parser.parse_known_args()
+
+    url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
+    key = args.apikey or os.getenv("DAILY_API_KEY")
+
+    if not url:
+        raise Exception(
+            "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL."
+        )
+
+    if not key:
+        raise Exception(
+            "No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers."
+        )
+
+    daily_rest_helper = DailyRESTHelper(
+        daily_api_key=key,
+        daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
+        aiohttp_session=aiohttp_session,
+    )
+
+    # Create a meeting token for the given room with an expiration 1 hour in
+    # the future.
+    expiry_time: float = 60 * 60
+
+    token = await daily_rest_helper.get_token(url, expiry_time)
+
+    return (url, token)

examples/fal-smart-turn/server/server.py

@@ -0,0 +1,228 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+"""RTVI Bot Server Implementation.
+
+This FastAPI server manages RTVI bot instances and provides endpoints for both
+direct browser access and RTVI client connections. It handles:
+- Creating Daily rooms
+- Managing bot processes
+- Providing connection credentials
+- Monitoring bot status
+
+Requirements:
+- Daily API key (set in .env file)
+- Python 3.10+
+- FastAPI
+- Running bot implementation
+"""
+
+import argparse
+import os
+import subprocess
+from contextlib import asynccontextmanager
+from typing import Any, Dict
+
+import aiohttp
+from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.responses import JSONResponse, RedirectResponse
+
+from pipecat.transports.services.helpers.daily_rest import DailyRESTHelper, DailyRoomParams
+
+# Load environment variables from .env file
+load_dotenv(override=True)
+
+# Maximum number of bot instances allowed per room
+MAX_BOTS_PER_ROOM = 1
+
+# Dictionary to track bot processes: {pid: (process, room_url)}
+bot_procs = {}
+
+# Store Daily API helpers
+daily_helpers = {}
+
+
+def cleanup():
+    """Cleanup function to terminate all bot processes.
+
+    Called during server shutdown.
+    """
+    for entry in bot_procs.values():
+        proc = entry[0]
+        proc.terminate()
+        proc.wait()
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """FastAPI lifespan manager that handles startup and shutdown tasks.
+
+    - Creates aiohttp session
+    - Initializes Daily API helper
+    - Cleans up resources on shutdown
+    """
+    aiohttp_session = aiohttp.ClientSession()
+    daily_helpers["rest"] = DailyRESTHelper(
+        daily_api_key=os.getenv("DAILY_API_KEY", ""),
+        daily_api_url=os.getenv("DAILY_API_URL", "https://api.daily.co/v1"),
+        aiohttp_session=aiohttp_session,
+    )
+    yield
+    await aiohttp_session.close()
+    cleanup()
+
+
+# Initialize FastAPI app with lifespan manager
+app = FastAPI(lifespan=lifespan)
+
+# Configure CORS to allow requests from any origin
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+async def create_room_and_token() -> tuple[str, str]:
+    """Helper function to create a Daily room and generate an access token.
+
+    Returns:
+        tuple[str, str]: A tuple containing (room_url, token)
+
+    Raises:
+        HTTPException: If room creation or token generation fails
+    """
+    room = await daily_helpers["rest"].create_room(DailyRoomParams())
+    if not room.url:
+        raise HTTPException(status_code=500, detail="Failed to create room")
+
+    token = await daily_helpers["rest"].get_token(room.url)
+    if not token:
+        raise HTTPException(status_code=500, detail=f"Failed to get token for room: {room.url}")
+
+    return room.url, token
+
+
+@app.get("/")
+async def start_agent(request: Request):
+    """Endpoint for direct browser access to the bot.
+
+    Creates a room, starts a bot instance, and redirects to the Daily room URL.
+
+    Returns:
+        RedirectResponse: Redirects to the Daily room URL
+
+    Raises:
+        HTTPException: If room creation, token generation, or bot startup fails
+    """
+    print("Creating room")
+    room_url, token = await create_room_and_token()
+    print(f"Room URL: {room_url}")
+
+    # Check if there is already an existing process running in this room
+    num_bots_in_room = sum(
+        1 for proc in bot_procs.values() if proc[1] == room_url and proc[0].poll() is None
+    )
+    if num_bots_in_room >= MAX_BOTS_PER_ROOM:
+        raise HTTPException(status_code=500, detail=f"Max bot limit reached for room: {room_url}")
+
+    # Spawn a new bot process
+    try:
+        proc = subprocess.Popen(
+            [f"python3 bot.py -u {room_url} -t {token}"],
+            shell=True,
+            bufsize=1,
+            cwd=os.path.dirname(os.path.abspath(__file__)),
+        )
+        bot_procs[proc.pid] = (proc, room_url)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
+
+    return RedirectResponse(room_url)
+
+
+@app.post("/connect")
+async def rtvi_connect(request: Request) -> Dict[Any, Any]:
+    """RTVI connect endpoint that creates a room and returns connection credentials.
+
+    This endpoint is called by RTVI clients to establish a connection.
+
+    Returns:
+        Dict[Any, Any]: Authentication bundle containing room_url and token
+
+    Raises:
+        HTTPException: If room creation, token generation, or bot startup fails
+    """
+    print("Creating room for RTVI connection")
+    room_url, token = await create_room_and_token()
+    print(f"Room URL: {room_url}")
+
+    # Start the bot process
+    try:
+        proc = subprocess.Popen(
+            [f"python3 -m bot -u {room_url} -t {token}"],
+            shell=True,
+            bufsize=1,
+            cwd=os.path.dirname(os.path.abspath(__file__)),
+        )
+        bot_procs[proc.pid] = (proc, room_url)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to start subprocess: {e}")
+
+    # Return the authentication bundle in format expected by DailyTransport
+    return {"room_url": room_url, "token": token}
+
+
+@app.get("/status/{pid}")
+def get_status(pid: int):
+    """Get the status of a specific bot process.
+
+    Args:
+        pid (int): Process ID of the bot
+
+    Returns:
+        JSONResponse: Status information for the bot
+
+    Raises:
+        HTTPException: If the specified bot process is not found
+    """
+    # Look up the subprocess
+    proc = bot_procs.get(pid)
+
+    # If the subprocess doesn't exist, return an error
+    if not proc:
+        raise HTTPException(status_code=404, detail=f"Bot with process id: {pid} not found")
+
+    # Check the status of the subprocess
+    status = "running" if proc[0].poll() is None else "finished"
+    return JSONResponse({"bot_id": pid, "status": status})
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    # Parse command line arguments for server configuration
+    default_host = os.getenv("HOST", "0.0.0.0")
+    default_port = int(os.getenv("FAST_API_PORT", "7860"))
+
+    parser = argparse.ArgumentParser(description="Daily Storyteller FastAPI server")
+    parser.add_argument("--host", type=str, default=default_host, help="Host address")
+    parser.add_argument("--port", type=int, default=default_port, help="Port number")
+    parser.add_argument("--reload", action="store_true", help="Reload code on change")
+
+    config = parser.parse_args()
+
+    # Start the FastAPI server
+    uvicorn.run(
+        "server:app",
+        host=config.host,
+        port=config.port,
+        reload=config.reload,
+    )