124 lines
3.7 KiB
Python
124 lines
3.7 KiB
Python
#
|
||
# Copyright (c) 2024–2025, Daily
|
||
#
|
||
# SPDX-License-Identifier: BSD 2-Clause License
|
||
#
|
||
|
||
import argparse
|
||
|
||
from dotenv import load_dotenv
|
||
from loguru import logger
|
||
|
||
from pipecat.frames.frames import (
|
||
EndFrame,
|
||
Frame,
|
||
TextFrame,
|
||
TTSTextFrame,
|
||
UserImageRequestFrame,
|
||
UserStartedSpeakingFrame,
|
||
)
|
||
from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
|
||
from pipecat.pipeline.pipeline import Pipeline
|
||
from pipecat.pipeline.runner import PipelineRunner
|
||
from pipecat.pipeline.task import PipelineTask
|
||
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
|
||
from pipecat.services.moondream.vision import MoondreamService
|
||
from pipecat.transports.base_input import BaseInputTransport
|
||
from pipecat.transports.base_output import BaseOutputTransport
|
||
from pipecat.transports.base_transport import TransportParams
|
||
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||
|
||
load_dotenv(override=True)
|
||
|
||
|
||
class UserImageRequester(FrameProcessor):
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.participant_id = None
|
||
|
||
def set_participant_id(self, participant_id: str):
|
||
self.participant_id = participant_id
|
||
|
||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||
await super().process_frame(frame, direction)
|
||
|
||
if self.participant_id and isinstance(frame, TextFrame):
|
||
await self.push_frame(
|
||
UserImageRequestFrame(self.participant_id), FrameDirection.UPSTREAM
|
||
)
|
||
await self.push_frame(
|
||
TextFrame(
|
||
"Is there a person wearing a blue shirt in this image? Only answer with YES or NO."
|
||
)
|
||
)
|
||
else:
|
||
await self.push_frame(frame, direction)
|
||
|
||
|
||
async def run_bot(webrtc_connection: SmallWebRTCConnection, args: argparse.Namespace):
|
||
logger.info(f"Starting bot with video input: {args.input}")
|
||
|
||
transport = SmallWebRTCTransport(
|
||
webrtc_connection=webrtc_connection,
|
||
params=TransportParams(
|
||
audio_out_enabled=True,
|
||
video_out_enabled=True,
|
||
video_out_is_live=True,
|
||
video_out_width=1280,
|
||
video_out_height=720,
|
||
),
|
||
)
|
||
|
||
gst = GStreamerPipelineSource(
|
||
pipeline=(f"rtspsrc location={args.input} ! decodebin ! autovideosink"),
|
||
out_params=GStreamerPipelineSource.OutputParams(
|
||
video_width=1280,
|
||
video_height=720,
|
||
),
|
||
)
|
||
|
||
# If you run into weird description, try with use_cpu=True
|
||
moondream = MoondreamService()
|
||
|
||
ir = UserImageRequester()
|
||
va = VisionImageFrameAggregator()
|
||
|
||
pipeline = Pipeline(
|
||
[
|
||
gst, # GStreamer file source
|
||
ir,
|
||
va,
|
||
moondream,
|
||
# alert_processor, # Send an email alert or something if the door is open
|
||
transport.output(), # Transport bot output
|
||
]
|
||
)
|
||
|
||
task = PipelineTask(
|
||
pipeline,
|
||
observers=[
|
||
DebugLogObserver(
|
||
frame_types={
|
||
TextFrame: None,
|
||
EndFrame: None,
|
||
}
|
||
),
|
||
],
|
||
)
|
||
|
||
runner = PipelineRunner(handle_sigint=False)
|
||
|
||
await runner.run(task)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
from run import main
|
||
|
||
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
|
||
parser.add_argument("-i", "--input", type=str, required=True, help="Input video file")
|
||
|
||
main(parser)
|