# # Copyright (c) 2024–2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # import os from typing import Optional from dotenv import load_dotenv from loguru import logger from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.frames.frames import Frame, TextFrame, TTSSpeakFrame, UserImageRequestFrame from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.user_response import UserResponseAggregator from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator from pipecat.processors.frame_processor import FrameDirection, FrameProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import ( create_transport, get_transport_client_id, maybe_capture_participant_camera, ) from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.google.llm import GoogleLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.services.daily import DailyParams load_dotenv(override=True) class UserImageRequester(FrameProcessor): def __init__(self, participant_id: Optional[str] = None): super().__init__() self._participant_id = participant_id def set_participant_id(self, participant_id: str): self._participant_id = participant_id async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) if self._participant_id and isinstance(frame, TextFrame): await self.push_frame( UserImageRequestFrame(self._participant_id), FrameDirection.UPSTREAM ) await self.push_frame(frame, direction) # We store functions so objects (e.g. SileroVADAnalyzer) don't get # instantiated. The function will be called when the desired transport gets # selected. transport_params = { "daily": lambda: DailyParams( audio_in_enabled=True, audio_out_enabled=True, video_in_enabled=True, vad_analyzer=SileroVADAnalyzer(), ), "webrtc": lambda: TransportParams( audio_in_enabled=True, audio_out_enabled=True, video_in_enabled=True, vad_analyzer=SileroVADAnalyzer(), ), } async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): logger.info(f"Starting bot") user_response = UserResponseAggregator() # Initialize the image requester without setting the participant ID yet image_requester = UserImageRequester() vision_aggregator = VisionImageFrameAggregator() stt = DeepgramSTTService(api_key=os.getenv("DEEPGRAM_API_KEY")) # Google Gemini model for vision analysis google = GoogleLLMService(model="gemini-2.0-flash-001", api_key=os.getenv("GOOGLE_API_KEY")) tts = CartesiaTTSService( api_key=os.getenv("CARTESIA_API_KEY"), voice_id="71a7ad14-091c-4e8e-a314-022ece01c121", # British Reading Lady ) pipeline = Pipeline( [ transport.input(), stt, user_response, image_requester, vision_aggregator, google, tts, transport.output(), ] ) task = PipelineTask( pipeline, params=PipelineParams( enable_metrics=True, enable_usage_metrics=True, ), idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, ) @transport.event_handler("on_client_connected") async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") await maybe_capture_participant_camera(transport, client) # Set the participant ID in the image requester client_id = get_transport_client_id(transport, client) image_requester.set_participant_id(client_id) # Welcome message await task.queue_frame(TTSSpeakFrame("Hi there! Feel free to ask me what I see.")) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): logger.info(f"Client disconnected") await task.cancel() runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) await runner.run(task) async def bot(runner_args: RunnerArguments): """Main bot entry point compatible with Pipecat Cloud.""" transport = await create_transport(runner_args, transport_params) await run_bot(transport, runner_args) if __name__ == "__main__": from pipecat.runner.run import main main()