162 lines
5.0 KiB
Python
162 lines
5.0 KiB
Python
#
|
||
# Copyright (c) 2024–2025, Daily
|
||
#
|
||
# SPDX-License-Identifier: BSD 2-Clause License
|
||
#
|
||
|
||
import argparse
|
||
from typing import Optional
|
||
|
||
from dotenv import load_dotenv
|
||
from loguru import logger
|
||
|
||
from pipecat.frames.frames import (
|
||
EndFrame,
|
||
Frame,
|
||
InputImageRawFrame,
|
||
OutputImageRawFrame,
|
||
TextFrame,
|
||
TTSTextFrame,
|
||
UserImageRequestFrame,
|
||
UserStartedSpeakingFrame,
|
||
)
|
||
from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
|
||
from pipecat.pipeline.pipeline import Pipeline
|
||
from pipecat.pipeline.runner import PipelineRunner
|
||
from pipecat.pipeline.task import PipelineTask
|
||
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
|
||
from pipecat.services.moondream.vision import MoondreamService
|
||
from pipecat.transports.base_input import BaseInputTransport
|
||
from pipecat.transports.base_output import BaseOutputTransport
|
||
from pipecat.transports.base_transport import TransportParams
|
||
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||
|
||
load_dotenv(override=True)
|
||
|
||
|
||
class AlertProcessor(FrameProcessor):
|
||
def __init__(self):
|
||
super().__init__()
|
||
|
||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||
await super().process_frame(frame, direction)
|
||
|
||
if isinstance(frame, TextFrame):
|
||
logger.info(f"Alert Processor received text: {frame.text}")
|
||
text = frame.text.strip().upper()
|
||
if text == "YES":
|
||
# SEND AN EMAIL HERE
|
||
logger.info("Alert: YES")
|
||
else:
|
||
logger.info("Alert: NO")
|
||
|
||
await self.push_frame(frame, direction)
|
||
|
||
|
||
class UserImageRequester(FrameProcessor):
|
||
def __init__(self, participant_id: Optional[str] = None):
|
||
super().__init__()
|
||
|
||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||
await super().process_frame(frame, direction)
|
||
|
||
if isinstance(frame, OutputImageRawFrame):
|
||
await self.push_frame(frame)
|
||
# logger.info(f"UserImageRequester received image frame with size: {frame.size}")
|
||
text_frame = TextFrame(
|
||
"Are there people in the bottom right corner of the image? Only answer with YES or NO."
|
||
)
|
||
await self.push_frame(text_frame)
|
||
input_frame = InputImageRawFrame(
|
||
image=frame.image,
|
||
size=frame.size,
|
||
format=frame.format,
|
||
)
|
||
await self.push_frame(input_frame)
|
||
else:
|
||
await self.push_frame(frame, direction)
|
||
|
||
|
||
async def run_bot(webrtc_connection: SmallWebRTCConnection, args: argparse.Namespace):
|
||
logger.info(f"Starting bot with video input: {args.input}")
|
||
|
||
transport = SmallWebRTCTransport(
|
||
webrtc_connection=webrtc_connection,
|
||
params=TransportParams(
|
||
audio_out_enabled=True,
|
||
video_out_enabled=True,
|
||
video_out_is_live=True,
|
||
video_out_width=1280,
|
||
video_out_height=720,
|
||
),
|
||
)
|
||
|
||
gst = GStreamerPipelineSource(
|
||
pipeline=(f"rtspsrc location={args.input} ! decodebin ! autovideosink"),
|
||
out_params=GStreamerPipelineSource.OutputParams(
|
||
video_width=1280,
|
||
video_height=720,
|
||
),
|
||
)
|
||
|
||
# If you run into weird description, try with use_cpu=True
|
||
moondream = MoondreamService()
|
||
|
||
ir = UserImageRequester()
|
||
va = VisionImageFrameAggregator()
|
||
alert = AlertProcessor()
|
||
|
||
pipeline = Pipeline(
|
||
[
|
||
gst, # GStreamer file source
|
||
ir,
|
||
# debug,
|
||
va,
|
||
moondream,
|
||
alert, # Send an email alert or something if the door is open
|
||
transport.output(), # Transport bot output
|
||
]
|
||
)
|
||
|
||
task = PipelineTask(
|
||
pipeline,
|
||
observers=[
|
||
DebugLogObserver(
|
||
frame_types={
|
||
# TextFrame: None,
|
||
TextFrame: (MoondreamService, FrameEndpoint.SOURCE),
|
||
# InputImageRawFrame: None,
|
||
EndFrame: None,
|
||
}
|
||
),
|
||
],
|
||
)
|
||
|
||
@transport.event_handler("on_client_connected")
|
||
async def on_client_connected(transport, client):
|
||
logger.info(f"Client connected: {client}")
|
||
|
||
await task.queue_frames(
|
||
[
|
||
TextFrame(
|
||
"Are there people in the bottom right corner of the image? Only answer with YES or NO."
|
||
)
|
||
]
|
||
)
|
||
|
||
runner = PipelineRunner(handle_sigint=False)
|
||
|
||
await runner.run(task)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
from run import main
|
||
|
||
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
|
||
parser.add_argument("-i", "--input", type=str, required=True, help="Input video file")
|
||
|
||
main(parser)
|