Fix 06a-image-sync.py

This commit is contained in:
Moishe Lettvin
2024-01-29 14:29:32 -05:00
parent 065a213ebb
commit d0bcddfd70
3 changed files with 56 additions and 57 deletions

View File

@@ -1,6 +1,8 @@
import argparse import argparse
import asyncio import asyncio
import os
from typing import AsyncGenerator from typing import AsyncGenerator
import aiohttp
import requests import requests
import time import time
import urllib.parse import urllib.parse
@@ -30,74 +32,71 @@ class ImageSyncAggregator(AIService):
async def main(room_url: str, token): async def main(room_url: str, token):
global transport async with aiohttp.ClientSession() as aiohttp_session:
global llm transport = DailyTransportService(
global tts room_url,
token,
transport = DailyTransportService( "Respond bot",
room_url, 5,
token,
"Respond bot",
5,
)
transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
transport.mic_enabled = True
transport.mic_sample_rate = 16000
llm = AzureLLMService()
tts = AzureTTSService()
img = FalImageGenService(image_size="1024x1024")
async def get_images():
get_speaking_task = asyncio.create_task(
img.run_image_gen("An image of a cat speaking")
)
get_waiting_task = asyncio.create_task(
img.run_image_gen("An image of a cat waiting")
) )
transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
transport.mic_enabled = True
transport.mic_sample_rate = 16000
(speaking_data, waiting_data) = await asyncio.gather( llm = AzureLLMService()
get_speaking_task, get_waiting_task tts = AzureTTSService()
) img = FalImageGenService(image_size="1024x1024", aiohttp_session=aiohttp_session)
return speaking_data, waiting_data async def get_images():
get_speaking_task = asyncio.create_task(
img.run_image_gen("An image of a cat speaking")
)
get_waiting_task = asyncio.create_task(
img.run_image_gen("An image of a cat waiting")
)
@transport.event_handler("on_first_other_participant_joined") (speaking_data, waiting_data) = await asyncio.gather(
async def on_first_other_participant_joined(transport): get_speaking_task, get_waiting_task
await tts.say("Hi, I'm listening!", transport.send_queue) )
async def handle_transcriptions(): return speaking_data, waiting_data
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMUserContextAggregator( @transport.event_handler("on_first_other_participant_joined")
messages, transport.my_participant_id async def on_first_other_participant_joined(transport):
) await tts.say("Hi, I'm listening!", transport.send_queue)
tma_out = LLMAssistantContextAggregator(
messages, transport.my_participant_id async def handle_transcriptions():
) messages = [
image_sync_aggregator = ImageSyncAggregator( {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
"/Users/moishe/src/daily-ai-sdk/src/samples/foundational/speaking.png", ]
"/Users/moishe/src/daily-ai-sdk/src/samples/foundational/waiting.png",
) tma_in = LLMUserContextAggregator(
await tts.run_to_queue( messages, transport.my_participant_id
transport.send_queue, )
image_sync_aggregator.run( tma_out = LLMAssistantContextAggregator(
tma_out.run( messages, transport.my_participant_id
llm.run( )
tma_in.run( image_sync_aggregator = ImageSyncAggregator(
transport.get_receive_frames() os.path.join(os.path.dirname(__file__), "images", "speaking.png"),
os.path.join(os.path.dirname(__file__), "images", "waiting.png"),
)
await tts.run_to_queue(
transport.send_queue,
image_sync_aggregator.run(
tma_out.run(
llm.run(
tma_in.run(
transport.get_receive_frames()
)
) )
) )
) )
) )
)
transport.transcription_settings["extra"]["punctuate"] = True transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions()) await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__": if __name__ == "__main__":

View File

Before

Width:  |  Height:  |  Size: 33 KiB

After

Width:  |  Height:  |  Size: 33 KiB

View File

Before

Width:  |  Height:  |  Size: 30 KiB

After

Width:  |  Height:  |  Size: 30 KiB