Fix 06a-image-sync.py
This commit is contained in:
@@ -1,6 +1,8 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import os
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
|
import aiohttp
|
||||||
import requests
|
import requests
|
||||||
import time
|
import time
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
@@ -30,74 +32,71 @@ class ImageSyncAggregator(AIService):
|
|||||||
|
|
||||||
|
|
||||||
async def main(room_url: str, token):
|
async def main(room_url: str, token):
|
||||||
global transport
|
async with aiohttp.ClientSession() as aiohttp_session:
|
||||||
global llm
|
transport = DailyTransportService(
|
||||||
global tts
|
room_url,
|
||||||
|
token,
|
||||||
transport = DailyTransportService(
|
"Respond bot",
|
||||||
room_url,
|
5,
|
||||||
token,
|
|
||||||
"Respond bot",
|
|
||||||
5,
|
|
||||||
)
|
|
||||||
transport.camera_enabled = True
|
|
||||||
transport.camera_width = 1024
|
|
||||||
transport.camera_height = 1024
|
|
||||||
transport.mic_enabled = True
|
|
||||||
transport.mic_sample_rate = 16000
|
|
||||||
|
|
||||||
llm = AzureLLMService()
|
|
||||||
tts = AzureTTSService()
|
|
||||||
img = FalImageGenService(image_size="1024x1024")
|
|
||||||
|
|
||||||
async def get_images():
|
|
||||||
get_speaking_task = asyncio.create_task(
|
|
||||||
img.run_image_gen("An image of a cat speaking")
|
|
||||||
)
|
|
||||||
get_waiting_task = asyncio.create_task(
|
|
||||||
img.run_image_gen("An image of a cat waiting")
|
|
||||||
)
|
)
|
||||||
|
transport.camera_enabled = True
|
||||||
|
transport.camera_width = 1024
|
||||||
|
transport.camera_height = 1024
|
||||||
|
transport.mic_enabled = True
|
||||||
|
transport.mic_sample_rate = 16000
|
||||||
|
|
||||||
(speaking_data, waiting_data) = await asyncio.gather(
|
llm = AzureLLMService()
|
||||||
get_speaking_task, get_waiting_task
|
tts = AzureTTSService()
|
||||||
)
|
img = FalImageGenService(image_size="1024x1024", aiohttp_session=aiohttp_session)
|
||||||
|
|
||||||
return speaking_data, waiting_data
|
async def get_images():
|
||||||
|
get_speaking_task = asyncio.create_task(
|
||||||
|
img.run_image_gen("An image of a cat speaking")
|
||||||
|
)
|
||||||
|
get_waiting_task = asyncio.create_task(
|
||||||
|
img.run_image_gen("An image of a cat waiting")
|
||||||
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_other_participant_joined")
|
(speaking_data, waiting_data) = await asyncio.gather(
|
||||||
async def on_first_other_participant_joined(transport):
|
get_speaking_task, get_waiting_task
|
||||||
await tts.say("Hi, I'm listening!", transport.send_queue)
|
)
|
||||||
|
|
||||||
async def handle_transcriptions():
|
return speaking_data, waiting_data
|
||||||
messages = [
|
|
||||||
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
|
|
||||||
]
|
|
||||||
|
|
||||||
tma_in = LLMUserContextAggregator(
|
@transport.event_handler("on_first_other_participant_joined")
|
||||||
messages, transport.my_participant_id
|
async def on_first_other_participant_joined(transport):
|
||||||
)
|
await tts.say("Hi, I'm listening!", transport.send_queue)
|
||||||
tma_out = LLMAssistantContextAggregator(
|
|
||||||
messages, transport.my_participant_id
|
async def handle_transcriptions():
|
||||||
)
|
messages = [
|
||||||
image_sync_aggregator = ImageSyncAggregator(
|
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
|
||||||
"/Users/moishe/src/daily-ai-sdk/src/samples/foundational/speaking.png",
|
]
|
||||||
"/Users/moishe/src/daily-ai-sdk/src/samples/foundational/waiting.png",
|
|
||||||
)
|
tma_in = LLMUserContextAggregator(
|
||||||
await tts.run_to_queue(
|
messages, transport.my_participant_id
|
||||||
transport.send_queue,
|
)
|
||||||
image_sync_aggregator.run(
|
tma_out = LLMAssistantContextAggregator(
|
||||||
tma_out.run(
|
messages, transport.my_participant_id
|
||||||
llm.run(
|
)
|
||||||
tma_in.run(
|
image_sync_aggregator = ImageSyncAggregator(
|
||||||
transport.get_receive_frames()
|
os.path.join(os.path.dirname(__file__), "images", "speaking.png"),
|
||||||
|
os.path.join(os.path.dirname(__file__), "images", "waiting.png"),
|
||||||
|
)
|
||||||
|
await tts.run_to_queue(
|
||||||
|
transport.send_queue,
|
||||||
|
image_sync_aggregator.run(
|
||||||
|
tma_out.run(
|
||||||
|
llm.run(
|
||||||
|
tma_in.run(
|
||||||
|
transport.get_receive_frames()
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
|
||||||
|
|
||||||
transport.transcription_settings["extra"]["punctuate"] = True
|
transport.transcription_settings["extra"]["punctuate"] = True
|
||||||
await asyncio.gather(transport.run(), handle_transcriptions())
|
await asyncio.gather(transport.run(), handle_transcriptions())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
|
Before Width: | Height: | Size: 33 KiB After Width: | Height: | Size: 33 KiB |
|
Before Width: | Height: | Size: 30 KiB After Width: | Height: | Size: 30 KiB |
Reference in New Issue
Block a user