From 2f80683dc4b311e9126772e718c0c524e8aba44f Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sat, 2 Nov 2024 10:42:31 -0700 Subject: [PATCH] initial commit of screen capture in 99-anthropic-hackathon.py --- .../foundational/99-anthropic-hackathon.py | 175 ++++++++++++++++++ src/pipecat/transports/services/daily.py | 7 +- 2 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 examples/foundational/99-anthropic-hackathon.py diff --git a/examples/foundational/99-anthropic-hackathon.py b/examples/foundational/99-anthropic-hackathon.py new file mode 100644 index 000000000..3478e9325 --- /dev/null +++ b/examples/foundational/99-anthropic-hackathon.py @@ -0,0 +1,175 @@ +# +# Copyright (c) 2024, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import asyncio +import aiohttp +import os +import sys + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.anthropic import AnthropicLLMService +from pipecat.transports.services.daily import DailyParams, DailyTransport +from pipecat.frames.frames import Frame, ImageRawFrame +from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.services.anthropic import AnthropicLLMContext + +from runner import configure + +from loguru import logger + +from dotenv import load_dotenv + +load_dotenv(override=True) + +logger.remove(0) +logger.add(sys.stderr, level="DEBUG") + +video_participant_id = None + +most_recent_image_frame = None + + +class ImageFrameCatcher(FrameProcessor): + async def process_frame(self, frame: Frame, direction: FrameDirection): + global most_recent_image_frame + + await super().process_frame(frame, direction) + if isinstance(frame, ImageRawFrame): + logger.debug(f"ImageLogger: {frame}") + most_recent_image_frame = frame + else: + await self.push_frame(frame, direction) + + +async def get_weather(function_name, tool_call_id, arguments, llm, context, result_callback): + location = arguments["location"] + await result_callback(f"The weather in {location} is currently 72 degrees and sunny.") + + +async def main(): + global llm + + async with aiohttp.ClientSession() as session: + (room_url, token) = await configure(session) + + transport = DailyTransport( + room_url, + token, + "Respond bot", + DailyParams( + audio_out_enabled=True, + transcription_enabled=True, + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + ) + + tts = CartesiaTTSService( + api_key=os.getenv("CARTESIA_API_KEY"), + voice_id="79a125e8-cd45-4c13-8a67-188112f4dd22", # British Lady + ) + + llm = AnthropicLLMService( + api_key=os.getenv("ANTHROPIC_API_KEY"), + model="claude-3-5-sonnet-20240620", + enable_prompt_caching_beta=True, + ) + llm.register_function("get_weather", get_weather) + + tools = [ + { + "name": "get_weather", + "description": "Get the current weather in a given location", + "input_schema": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + } + }, + "required": ["location"], + }, + }, + ] + + # todo: test with very short initial user message + + system_prompt = """\ +You are a helpful assistant who converses with a user and answers questions. Respond concisely to general questions. Keep +your answers brief unless explicitly asked for more information. + +Your response will be turned into speech so use only simple words and punctuation. + """ + + messages = [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": system_prompt, + } + ], + }, + {"role": "user", "content": "Start the conversation by saying 'hello'."}, + ] + + context = OpenAILLMContext(messages, tools) + context_aggregator = llm.create_context_aggregator(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + ImageFrameCatcher(), + context_aggregator.user(), # User speech to text + llm, # LLM + tts, # TTS + transport.output(), # Transport bot output + context_aggregator.assistant(), # Assistant spoken responses and tool context + ] + ) + + task = PipelineTask(pipeline, PipelineParams(allow_interruptions=True, enable_metrics=True)) + + @transport.event_handler("on_first_participant_joined") + async def on_first_participant_joined(transport, participant): + global video_participant_id + video_participant_id = participant["id"] + await transport.capture_participant_transcription(video_participant_id) + await transport.capture_participant_video( + video_participant_id, framerate=1, video_source="screenVideo" + ) + # Kick off the conversation. + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + @transport.event_handler("on_app_message") + async def on_app_message(transport, message, sender): + c = AnthropicLLMContext.upgrade_to_anthropic(context) + logger.debug(f"Received app message: {message} - {context}") + frame = most_recent_image_frame + if not frame: + logger.debug("No image frame to send") + return + c.add_image_frame_message( + format=frame.format, + size=frame.size, + image=frame.image, + text=message["message"], + ) + await task.queue_frames([context_aggregator.user().get_context_frame()]) + + runner = PipelineRunner() + await runner.run(task) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/pipecat/transports/services/daily.py b/src/pipecat/transports/services/daily.py index 7f64d1f00..37093b920 100644 --- a/src/pipecat/transports/services/daily.py +++ b/src/pipecat/transports/services/daily.py @@ -495,9 +495,12 @@ class DailyTransportClient(EventHandler): video_source: str = "camera", color_format: str = "RGB", ): - # Only enable camera subscription on this participant + # Try to enable camera and screen subscription on this participant await self.update_subscriptions( - participant_settings={participant_id: {"media": "subscribed"}} + # participant_settings={participant_id: {"media": "subscribed"}} + participant_settings={ + participant_id: {"media": {"camera": "subscribed", "screenVideo": "subscribed"}} + } ) self._video_renderers[participant_id] = callback