From d0bcddfd702333241b584adbb494cfa832806503 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Mon, 29 Jan 2024 14:29:32 -0500 Subject: [PATCH] Fix 06a-image-sync.py --- src/samples/foundational/06a-image-sync.py | 113 +++++++++--------- .../foundational/{ => images}/speaking.png | Bin .../foundational/{ => images}/waiting.png | Bin 3 files changed, 56 insertions(+), 57 deletions(-) rename src/samples/foundational/{ => images}/speaking.png (100%) rename src/samples/foundational/{ => images}/waiting.png (100%) diff --git a/src/samples/foundational/06a-image-sync.py b/src/samples/foundational/06a-image-sync.py index ede4d7648..5967cecaa 100644 --- a/src/samples/foundational/06a-image-sync.py +++ b/src/samples/foundational/06a-image-sync.py @@ -1,6 +1,8 @@ import argparse import asyncio +import os from typing import AsyncGenerator +import aiohttp import requests import time import urllib.parse @@ -30,74 +32,71 @@ class ImageSyncAggregator(AIService): async def main(room_url: str, token): - global transport - global llm - global tts - - transport = DailyTransportService( - room_url, - token, - "Respond bot", - 5, - ) - transport.camera_enabled = True - transport.camera_width = 1024 - transport.camera_height = 1024 - transport.mic_enabled = True - transport.mic_sample_rate = 16000 - - llm = AzureLLMService() - tts = AzureTTSService() - img = FalImageGenService(image_size="1024x1024") - - async def get_images(): - get_speaking_task = asyncio.create_task( - img.run_image_gen("An image of a cat speaking") - ) - get_waiting_task = asyncio.create_task( - img.run_image_gen("An image of a cat waiting") + async with aiohttp.ClientSession() as aiohttp_session: + transport = DailyTransportService( + room_url, + token, + "Respond bot", + 5, ) + transport.camera_enabled = True + transport.camera_width = 1024 + transport.camera_height = 1024 + transport.mic_enabled = True + transport.mic_sample_rate = 16000 - (speaking_data, waiting_data) = await asyncio.gather( - get_speaking_task, get_waiting_task - ) + llm = AzureLLMService() + tts = AzureTTSService() + img = FalImageGenService(image_size="1024x1024", aiohttp_session=aiohttp_session) - return speaking_data, waiting_data + async def get_images(): + get_speaking_task = asyncio.create_task( + img.run_image_gen("An image of a cat speaking") + ) + get_waiting_task = asyncio.create_task( + img.run_image_gen("An image of a cat waiting") + ) - @transport.event_handler("on_first_other_participant_joined") - async def on_first_other_participant_joined(transport): - await tts.say("Hi, I'm listening!", transport.send_queue) + (speaking_data, waiting_data) = await asyncio.gather( + get_speaking_task, get_waiting_task + ) - async def handle_transcriptions(): - messages = [ - {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, - ] + return speaking_data, waiting_data - tma_in = LLMUserContextAggregator( - messages, transport.my_participant_id - ) - tma_out = LLMAssistantContextAggregator( - messages, transport.my_participant_id - ) - image_sync_aggregator = ImageSyncAggregator( - "/Users/moishe/src/daily-ai-sdk/src/samples/foundational/speaking.png", - "/Users/moishe/src/daily-ai-sdk/src/samples/foundational/waiting.png", - ) - await tts.run_to_queue( - transport.send_queue, - image_sync_aggregator.run( - tma_out.run( - llm.run( - tma_in.run( - transport.get_receive_frames() + @transport.event_handler("on_first_other_participant_joined") + async def on_first_other_participant_joined(transport): + await tts.say("Hi, I'm listening!", transport.send_queue) + + async def handle_transcriptions(): + messages = [ + {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, + ] + + tma_in = LLMUserContextAggregator( + messages, transport.my_participant_id + ) + tma_out = LLMAssistantContextAggregator( + messages, transport.my_participant_id + ) + image_sync_aggregator = ImageSyncAggregator( + os.path.join(os.path.dirname(__file__), "images", "speaking.png"), + os.path.join(os.path.dirname(__file__), "images", "waiting.png"), + ) + await tts.run_to_queue( + transport.send_queue, + image_sync_aggregator.run( + tma_out.run( + llm.run( + tma_in.run( + transport.get_receive_frames() + ) ) ) ) ) - ) - transport.transcription_settings["extra"]["punctuate"] = True - await asyncio.gather(transport.run(), handle_transcriptions()) + transport.transcription_settings["extra"]["punctuate"] = True + await asyncio.gather(transport.run(), handle_transcriptions()) if __name__ == "__main__": diff --git a/src/samples/foundational/speaking.png b/src/samples/foundational/images/speaking.png similarity index 100% rename from src/samples/foundational/speaking.png rename to src/samples/foundational/images/speaking.png diff --git a/src/samples/foundational/waiting.png b/src/samples/foundational/images/waiting.png similarity index 100% rename from src/samples/foundational/waiting.png rename to src/samples/foundational/images/waiting.png