add 47-custom-frame-processor.py to foundational examples

examples/foundational/47-custom-frame-processor.py

@@ -0,0 +1,223 @@
+#
+# Copyright (c) 2024–2025, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import io
+import os
+import re
+
+from dotenv import load_dotenv
+from loguru import logger
+
+from pipecat.audio.turn.smart_turn.base_smart_turn import SmartTurnParams
+from pipecat.audio.turn.smart_turn.local_smart_turn_v3 import LocalSmartTurnAnalyzerV3
+from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
+from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    CancelFrame,
+    EndFrame,
+    Frame,
+    FunctionCallResultFrame,
+    InputAudioRawFrame,
+    InterruptionFrame,
+    LLMRunFrame,
+    LLMTextFrame,
+    StartFrame,
+    UserStartedSpeakingFrame,
+    UserStoppedSpeakingFrame,
+    VADUserStartedSpeakingFrame,
+)
+from pipecat.pipeline.pipeline import Pipeline
+from pipecat.pipeline.runner import PipelineRunner
+from pipecat.pipeline.task import PipelineParams, PipelineTask
+from pipecat.processors.aggregators.llm_context import LLMContext
+from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair
+from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
+from pipecat.runner.types import RunnerArguments
+from pipecat.runner.utils import create_transport
+from pipecat.services.cartesia.tts import CartesiaTTSService
+from pipecat.services.deepgram.stt import DeepgramSTTService
+from pipecat.services.openai.llm import OpenAILLMService
+from pipecat.transports.base_transport import BaseTransport, TransportParams
+from pipecat.transports.daily.transport import DailyParams
+
+load_dotenv(override=True)
+
+
+class CustomFrameProcessor(FrameProcessor):
+    """CustomFrameProcessor does 3 things:
+
+    1. keeps count of `InputAudioRawFrame` frames and logs count
+    when a `UserStoppedSpeakingFrame` is emitted.
+
+    2. Filters `LLMTextFrame` frames and replaces "the" with "the pumpkin".
+
+    3. Logs the following frames:
+        BotStartedSpeakingFrame
+        BotStoppedSpeakingFrame
+        CancelFrame
+        EndFrame
+        InterruptionFrame
+        StartFrame
+        UserStartedSpeakingFrame
+        VADUserStartedSpeakingFrame
+
+    4. Always pushes all frames
+
+    """
+
+    def __init__(self):
+        super().__init__()
+        self._raw_audio_input_frame_count = 0
+
+    async def process_frame(self, frame: Frame, direction: FrameDirection):
+        await super().process_frame(frame, direction)
+
+        #### 1.
+        # InputAudioRawFrames are noisy- probably don't want to log every instance
+        # keep a count and only log it when we see `UserStoppedSpeakingFrame`
+        if isinstance(frame, InputAudioRawFrame):
+            self._raw_audio_input_frame_count = self._raw_audio_input_frame_count + 1
+            await self.push_frame(frame, direction)
+
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            logger.info(
+                f"* * frame: {frame}; number of `InputAudioRawFrame` frames so far: {self._raw_audio_input_frame_count}"
+            )
+            await self.push_frame(frame, direction)
+
+        #### 2.
+        # everytime the LLM's response includes "the", replace it with "the pumpkin"
+        elif isinstance(frame, LLMTextFrame):
+            if "the" in frame.text:
+                text = re.sub(r" the\b", " the pumpkin", frame.text)
+                frame.text = text
+            await self.push_frame(frame, direction)
+
+        #### 3.
+        # frames types to log
+        elif isinstance(
+            frame,
+            (
+                BotStartedSpeakingFrame,
+                BotStoppedSpeakingFrame,
+                CancelFrame,
+                EndFrame,
+                InterruptionFrame,
+                StartFrame,
+                UserStartedSpeakingFrame,
+                VADUserStartedSpeakingFrame,
+            ),
+        ):
+            logger.info(f"* * frame: {frame}")
+            await self.push_frame(frame, direction)
+
+        #### 4.
+        # ALWAYS push all other frames
+        else:
+            # SUPER IMPORTANT: always push every frame!
+            await self.push_frame(frame, direction)
+
+
+# We store functions so objects (e.g. SileroVADAnalyzer) don't get
+# instantiated. The function will be called when the desired transport gets
+# selected.
+transport_params = {
+    "daily": lambda: DailyParams(
+        audio_in_enabled=True,
+        audio_out_enabled=True,
+        vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+        turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+    ),
+    "webrtc": lambda: TransportParams(
+        audio_in_enabled=True,
+        audio_out_enabled=True,
+        video_out_enabled=True,
+        vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
+        turn_analyzer=LocalSmartTurnAnalyzerV3(params=SmartTurnParams()),
+    ),
+}
+
+
+async def run_bot(transport: BaseTransport, runner_args: RunnerArguments):
+    logger.info(f"Starting bot")
+
+    stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY"))
+
+    tts = CartesiaTTSService(
+        api_key=os.getenv("CARTESIA_API_KEY"),
+        voice_id="71a7ad14-091c-4e8e-a314-022ece01c121",  # British Reading Lady
+    )
+
+    llm = OpenAILLMService(api_key=os.getenv("OPENAI_API_KEY"))
+
+    custom_frame_processor = CustomFrameProcessor()
+
+    messages = [
+        {
+            "role": "system",
+            "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio so don't include special characters in your answers. Respond to what the user said in a creative and helpful way.",
+        },
+    ]
+
+    context = LLMContext(messages)
+    context_aggregator = LLMContextAggregatorPair(context)
+
+    pipeline = Pipeline(
+        [
+            transport.input(),
+            stt,
+            context_aggregator.user(),
+            llm,
+            custom_frame_processor,  # filter and log frames
+            tts,
+            transport.output(),
+            context_aggregator.assistant(),
+        ]
+    )
+
+    task = PipelineTask(
+        pipeline,
+        params=PipelineParams(
+            enable_metrics=True,
+            enable_usage_metrics=True,
+        ),
+        idle_timeout_secs=runner_args.pipeline_idle_timeout_secs,
+    )
+
+    @transport.event_handler("on_client_connected")
+    async def on_client_connected(transport, client):
+        logger.info(f"Client connected: {client}")
+        # Kick off the conversation.
+        messages.append(
+            {
+                "role": "system",
+                "content": "Please introduce yourself to the user and inform them that your responses illustrate use of a Custom Frame Processor.",
+            }
+        )
+        await task.queue_frames([LLMRunFrame()])
+
+    @transport.event_handler("on_client_disconnected")
+    async def on_client_disconnected(transport, client):
+        logger.info(f"Client disconnected")
+        await task.cancel()
+
+    runner = PipelineRunner(handle_sigint=runner_args.handle_sigint)
+
+    await runner.run(task)
+
+
+async def bot(runner_args: RunnerArguments):
+    """Main bot entry point compatible with Pipecat Cloud."""
+    transport = await create_transport(runner_args, transport_params)
+    await run_bot(transport, runner_args)
+
+
+if __name__ == "__main__":
+    from pipecat.runner.run import main
+
+    main()