Compare commits
10 Commits
hush/TurnT
...
hush/rtsp
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
872f2ba71d | ||
|
|
b71b6fa487 | ||
|
|
d4de9cbf34 | ||
|
|
3335629767 | ||
|
|
ce23a84ceb | ||
|
|
887cdb4057 | ||
|
|
9ac077a6b2 | ||
|
|
b44c9574e8 | ||
|
|
dda515bf6a | ||
|
|
a9cb488bdd |
165
examples/foundational/18b-gstreamer.py
Normal file
165
examples/foundational/18b-gstreamer.py
Normal file
@@ -0,0 +1,165 @@
|
||||
#
|
||||
# Copyright (c) 2024–2025, Daily
|
||||
#
|
||||
# SPDX-License-Identifier: BSD 2-Clause License
|
||||
#
|
||||
|
||||
import argparse
|
||||
from typing import Optional
|
||||
|
||||
from dotenv import load_dotenv
|
||||
from loguru import logger
|
||||
|
||||
from pipecat.frames.frames import (
|
||||
EndFrame,
|
||||
Frame,
|
||||
InputImageRawFrame,
|
||||
OutputImageRawFrame,
|
||||
TextFrame,
|
||||
TTSTextFrame,
|
||||
UserImageRequestFrame,
|
||||
UserStartedSpeakingFrame,
|
||||
)
|
||||
from pipecat.observers.loggers.debug_log_observer import DebugLogObserver, FrameEndpoint
|
||||
from pipecat.pipeline.pipeline import Pipeline
|
||||
from pipecat.pipeline.runner import PipelineRunner
|
||||
from pipecat.pipeline.task import PipelineTask
|
||||
from pipecat.processors.aggregators.vision_image_frame import VisionImageFrameAggregator
|
||||
from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
|
||||
from pipecat.processors.frameworks.rtvi import (
|
||||
RTVIConfig,
|
||||
RTVIObserver,
|
||||
RTVIProcessor,
|
||||
RTVIServerMessageFrame,
|
||||
)
|
||||
from pipecat.processors.gstreamer.pipeline_source import GStreamerPipelineSource
|
||||
from pipecat.services.moondream.vision import MoondreamService
|
||||
from pipecat.transports.base_input import BaseInputTransport
|
||||
from pipecat.transports.base_output import BaseOutputTransport
|
||||
from pipecat.transports.base_transport import TransportParams
|
||||
from pipecat.transports.network.small_webrtc import SmallWebRTCTransport
|
||||
from pipecat.transports.network.webrtc_connection import SmallWebRTCConnection
|
||||
|
||||
load_dotenv(override=True)
|
||||
|
||||
|
||||
class AlertProcessor(FrameProcessor):
|
||||
def __init__(self, connection: SmallWebRTCConnection):
|
||||
super().__init__()
|
||||
self._connection = connection
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, TextFrame):
|
||||
text = frame.text.strip().upper()
|
||||
message_frame = RTVIServerMessageFrame(data=text)
|
||||
await self.push_frame(message_frame)
|
||||
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
|
||||
class UserImageRequester(FrameProcessor):
|
||||
def __init__(self, participant_id: Optional[str] = None):
|
||||
super().__init__()
|
||||
|
||||
async def process_frame(self, frame: Frame, direction: FrameDirection):
|
||||
await super().process_frame(frame, direction)
|
||||
|
||||
if isinstance(frame, OutputImageRawFrame):
|
||||
await self.push_frame(frame)
|
||||
# logger.info(f"UserImageRequester received image frame with size: {frame.size}")
|
||||
text_frame = TextFrame(
|
||||
"Are there people in the bottom right corner of the image? Only answer with YES or NO."
|
||||
)
|
||||
await self.push_frame(text_frame)
|
||||
input_frame = InputImageRawFrame(
|
||||
image=frame.image,
|
||||
size=frame.size,
|
||||
format=frame.format,
|
||||
)
|
||||
await self.push_frame(input_frame)
|
||||
else:
|
||||
await self.push_frame(frame, direction)
|
||||
|
||||
|
||||
async def run_bot(webrtc_connection: SmallWebRTCConnection, args: argparse.Namespace):
|
||||
logger.info(f"Starting bot with video input: {args.input}")
|
||||
|
||||
transport = SmallWebRTCTransport(
|
||||
webrtc_connection=webrtc_connection,
|
||||
params=TransportParams(
|
||||
audio_out_enabled=True,
|
||||
video_out_enabled=True,
|
||||
video_out_is_live=True,
|
||||
video_out_width=1280,
|
||||
video_out_height=720,
|
||||
),
|
||||
)
|
||||
|
||||
gst = GStreamerPipelineSource(
|
||||
pipeline=(f"rtspsrc location={args.input} ! decodebin ! autovideosink"),
|
||||
out_params=GStreamerPipelineSource.OutputParams(
|
||||
video_width=1280,
|
||||
video_height=720,
|
||||
),
|
||||
)
|
||||
|
||||
rtvi = RTVIProcessor(config=RTVIConfig(config=[]))
|
||||
|
||||
# If you run into weird description, try with use_cpu=True
|
||||
moondream = MoondreamService()
|
||||
|
||||
ir = UserImageRequester()
|
||||
va = VisionImageFrameAggregator()
|
||||
alert = AlertProcessor(connection=webrtc_connection)
|
||||
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
gst, # GStreamer file source
|
||||
rtvi,
|
||||
ir,
|
||||
# debug,
|
||||
va,
|
||||
moondream,
|
||||
alert, # Send an email alert or something if the door is open
|
||||
transport.output(), # Transport bot output
|
||||
]
|
||||
)
|
||||
|
||||
task = PipelineTask(
|
||||
pipeline,
|
||||
observers=[
|
||||
RTVIObserver(rtvi),
|
||||
DebugLogObserver(
|
||||
frame_types={
|
||||
# TextFrame: None,
|
||||
TextFrame: (MoondreamService, FrameEndpoint.SOURCE),
|
||||
# InputImageRawFrame: None,
|
||||
EndFrame: None,
|
||||
}
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
@rtvi.event_handler("on_client_ready")
|
||||
async def on_client_ready(rtvi):
|
||||
logger.info(f"Bot ready: {rtvi}")
|
||||
await rtvi.set_bot_ready()
|
||||
|
||||
@transport.event_handler("on_client_connected")
|
||||
async def on_client_connected(transport, client):
|
||||
logger.info(f"Client connected: {client}")
|
||||
|
||||
runner = PipelineRunner(handle_sigint=False)
|
||||
|
||||
await runner.run(task)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from run import main
|
||||
|
||||
parser = argparse.ArgumentParser(description="Pipecat Bot Runner")
|
||||
parser.add_argument("-i", "--input", type=str, required=True, help="Input video file")
|
||||
|
||||
main(parser)
|
||||
Reference in New Issue
Block a user