wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

examples: capture camera or screen. allow setting framerate

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2025-05-29 10:15:18 -07:00
parent 67dd146038
commit ca7b0650c2
11 changed files with 34 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/foundational/09a-local-mirror.py
View File

@@ -11,6 +11,7 @@ import tkinter as tk
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from pipecat.examples.run import maybe_capture_participant_camera
from pipecat.frames.frames import ( from pipecat.frames.frames import (
Frame, Frame,
InputAudioRawFrame, InputAudioRawFrame,
@@ -107,6 +108,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
@transport.event_handler("on_client_connected") @transport.event_handler("on_client_connected")
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected") logger.info(f"Client connected")
await maybe_capture_participant_camera(transport, client, framerate=30)
@transport.event_handler("on_client_disconnected") @transport.event_handler("on_client_disconnected")
async def on_client_disconnected(transport, client): async def on_client_disconnected(transport, client):

4
examples/foundational/12-describe-video.py
View File

@@ -12,7 +12,7 @@ from dotenv import load_dotenv
from loguru import logger from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -105,7 +105,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
# Set the participant ID in the image requester # Set the participant ID in the image requester
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

4
examples/foundational/12a-describe-video-gemini-flash.py
View File

@@ -12,7 +12,7 @@ from dotenv import load_dotenv
from loguru import logger from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -108,7 +108,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
# Set the participant ID in the image requester # Set the participant ID in the image requester
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

4
examples/foundational/12b-describe-video-gpt-4o.py
View File

@@ -12,7 +12,7 @@ from dotenv import load_dotenv
from loguru import logger from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -108,7 +108,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
# Set the participant ID in the image requester # Set the participant ID in the image requester
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

4
examples/foundational/12c-describe-video-anthropic.py
View File

@@ -12,7 +12,7 @@ from dotenv import load_dotenv
from loguru import logger from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame from pipecat.frames.frames import Frame, TextFrame, UserImageRequestFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -108,7 +108,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
# Set the participant ID in the image requester # Set the participant ID in the image requester
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

4
examples/foundational/14b-function-calling-anthropic-video.py
View File

@@ -14,7 +14,7 @@ from loguru import logger
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -181,7 +181,7 @@ If you need to use a tool, simply use the tool. Do not tell the user the tool yo
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
global client_id global client_id
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

4
examples/foundational/14d-function-calling-video.py
View File

@@ -14,7 +14,7 @@ from loguru import logger
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineTask from pipecat.pipeline.task import PipelineTask
@@ -164,7 +164,7 @@ indicate you should use the get_image tool are:
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected") logger.info(f"Client connected")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
global client_id global client_id
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

4
examples/foundational/14e-function-calling-gemini.py
View File

@@ -14,7 +14,7 @@ from loguru import logger
from pipecat.adapters.schemas.function_schema import FunctionSchema from pipecat.adapters.schemas.function_schema import FunctionSchema
from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.schemas.tools_schema import ToolsSchema
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.frames.frames import TTSSpeakFrame from pipecat.frames.frames import TTSSpeakFrame
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
@@ -174,7 +174,7 @@ indicate you should use the get_image tool are:
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
global client_id global client_id
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

4
examples/foundational/20d-persistent-context-gemini.py
View File

@@ -15,7 +15,7 @@ from loguru import logger
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_video from pipecat.examples.run import get_transport_client_id, maybe_capture_participant_camera
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -286,7 +286,7 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected") logger.info(f"Client connected")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client)
global client_id global client_id
client_id = get_transport_client_id(transport, client) client_id = get_transport_client_id(transport, client)

5
examples/foundational/26c-gemini-multimodal-live-video.py
View File

@@ -10,10 +10,10 @@ import os
from dotenv import load_dotenv from dotenv import load_dotenv
from loguru import logger from loguru import logger
from run import maybe_capture_participant_video
from pipecat.audio.vad.silero import SileroVADAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.examples.run import maybe_capture_participant_camera, maybe_capture_participant_screen
from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -92,7 +92,8 @@ async def run_example(transport: BaseTransport, _: argparse.Namespace, handle_si
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_video(transport, client) await maybe_capture_participant_camera(transport, client, framerate=1)
await maybe_capture_participant_screen(transport, client, framerate=1)
await task.queue_frames([context_aggregator.user().get_context_frame()]) await task.queue_frames([context_aggregator.user().get_context_frame()])
await asyncio.sleep(3) await asyncio.sleep(3)

16
src/pipecat/examples/run.py
View File

@@ -43,11 +43,21 @@ def get_transport_client_id(transport: BaseTransport, client: Any) -> str:
return "" return ""
async def maybe_capture_participant_video(transport: BaseTransport, client: Any): async def maybe_capture_participant_camera(
transport: BaseTransport, client: Any, framerate: int = 0
):
if isinstance(transport, DailyTransport): if isinstance(transport, DailyTransport):
await transport.capture_participant_video(client["id"], framerate=0, video_source="camera")
await transport.capture_participant_video( await transport.capture_participant_video(
client["id"], framerate=0, video_source="screenVideo" client["id"], framerate=framerate, video_source="camera"
)
async def maybe_capture_participant_screen(
transport: BaseTransport, client: Any, framerate: int = 0
):
if isinstance(transport, DailyTransport):
await transport.capture_participant_video(
client["id"], framerate=framerate, video_source="screenVideo"
) )