Compare commits

...

10 Commits

Author SHA1 Message Date
James Hush
872f2ba71d Remove log 2025-05-25 15:17:16 +08:00
James Hush
b71b6fa487 Send message to client 2025-05-25 15:17:05 +08:00
James Hush
d4de9cbf34 Show email 2025-05-25 11:28:52 +08:00
James Hush
3335629767 This works 2025-05-25 11:27:25 +08:00
James Hush
ce23a84ceb Get alerts working 2025-05-25 11:11:17 +08:00
James Hush
887cdb4057 Get detection working 2025-05-25 11:02:14 +08:00
James Hush
9ac077a6b2 Save 2025-05-25 09:30:41 +08:00
James Hush
b44c9574e8 Save log observer 2025-05-23 09:53:37 +08:00
James Hush
dda515bf6a Get moondream glue 2025-05-23 09:44:41 +08:00
James Hush
a9cb488bdd Get it working with url 2025-05-23 09:16:42 +08:00

View File

@@ -0,0 +1,165 @@
#
# Copyright (c) 20242025, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import argparse
from typing import Optional
from dotenv import load_dotenv
from loguru import logger
from pipecat.frames.frames import (
EndFrame,
Frame,
InputImageRawFrame,
OutputImageRawFrame,
TextFrame,
TTSTextFrame,
UserImageRequestFrame,
UserStartedSpeakingFrame,
)
from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
from pipecat.processors.frameworks.rtvi import (
RTVIConfig,
RTVIObserver,
RTVIProcessor,
RTVIServerMessageFrame,
)
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
from pipecat.services.moondream.vision import MoondreamService
from pipecat.transports.base_input import BaseInputTransport
from pipecat.transports.base_output import BaseOutputTransport
from pipecat.transports.base_transport import TransportParams
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
load_dotenv(override=True)
class AlertProcessor(FrameProcessor):
def __init__(self, connection: SmallWebRTCConnection):
super().__init__()
self._connection = connection
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, TextFrame):
text = frame.text.strip().upper()
message_frame = RTVIServerMessageFrame(data=text)
await self.push_frame(message_frame)
await self.push_frame(frame, direction)
class UserImageRequester(FrameProcessor):
def __init__(self, participant_id: Optional[str] = None):
super().__init__()
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, OutputImageRawFrame):
await self.push_frame(frame)
# logger.info(f"UserImageRequester received image frame with size: {frame.size}")
text_frame = TextFrame(
"Are there people in the bottom right corner of the image? Only answer with YES or NO."
)
await self.push_frame(text_frame)
input_frame = InputImageRawFrame(
image=frame.image,
size=frame.size,
format=frame.format,
)
await self.push_frame(input_frame)
else:
await self.push_frame(frame, direction)
async def run_bot(webrtc_connection: SmallWebRTCConnection, args: argparse.Namespace):
logger.info(f"Starting bot with video input: {args.input}")
transport = SmallWebRTCTransport(
webrtc_connection=webrtc_connection,
params=TransportParams(
audio_out_enabled=True,
video_out_enabled=True,
video_out_is_live=True,
video_out_width=1280,
video_out_height=720,
),
)
gst = GStreamerPipelineSource(
pipeline=(f"rtspsrc location={args.input} ! decodebin ! autovideosink"),
out_params=GStreamerPipelineSource.OutputParams(
video_width=1280,
video_height=720,
),
)
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
# If you run into weird description, try with use_cpu=True
moondream = MoondreamService()
ir = UserImageRequester()
va = VisionImageFrameAggregator()
alert = AlertProcessor(connection=webrtc_connection)
pipeline = Pipeline(
[
gst, # GStreamer file source
rtvi,
ir,
# debug,
va,
moondream,
alert, # Send an email alert or something if the door is open
transport.output(), # Transport bot output
]
)
task = PipelineTask(
pipeline,
observers=[
RTVIObserver(rtvi),
DebugLogObserver(
frame_types={
# TextFrame: None,
TextFrame: (MoondreamService, FrameEndpoint.SOURCE),
# InputImageRawFrame: None,
EndFrame: None,
}
),
],
)
@rtvi.event_handler("on_client_ready")
async def on_client_ready(rtvi):
logger.info(f"Bot ready: {rtvi}")
await rtvi.set_bot_ready()
@transport.event_handler("on_client_connected")
async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}")
runner = PipelineRunner(handle_sigint=False)
await runner.run(task)
if __name__ == "__main__":
from run import main
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
parser.add_argument("-i", "--input", type=str, required=True, help="Input video file")
main(parser)