From cd204ebd21d3c6d88ccae7cff89f2a8562cdf4b5 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Tue, 9 Jan 2024 14:19:27 -0500 Subject: [PATCH] Add sample 04- --- .../04-utterance-and-speech.py | 53 +++++--- src/samples/theoretical-to-real/05-queued.py | 119 ++++++++++++++++++ .../05-sync-speech-and-text.py | 1 + 3 files changed, 156 insertions(+), 17 deletions(-) create mode 100644 src/samples/theoretical-to-real/05-queued.py diff --git a/src/samples/theoretical-to-real/04-utterance-and-speech.py b/src/samples/theoretical-to-real/04-utterance-and-speech.py index 1ec5dccfb..880845834 100644 --- a/src/samples/theoretical-to-real/04-utterance-and-speech.py +++ b/src/samples/theoretical-to-real/04-utterance-and-speech.py @@ -1,33 +1,52 @@ import argparse import asyncio +import re -from dailyai.output_queue import OutputQueueFrame, FrameType from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.open_ai_services import OpenAIImageGenService +from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.output_queue import OutputQueueFrame, FrameType -local_joined = False -participant_joined = False +async def main(room_url:str): + global transport + global llm + global tts -async def main(room_url): - meeting_duration_minutes = 1 transport = DailyTransportService( room_url, None, - "Show a still frame image", - meeting_duration_minutes, + "Say Two Things Bot", + 1, ) - transport.mic_enabled = False - transport.camera_enabled = True - transport.camera_width = 1024 - transport.camera_height = 1024 + transport.mic_enabled = True + transport.mic_sample_rate = 16000 + transport.camera_enabled = False - imagegen = OpenAIImageGenService() - image_task = asyncio.create_task(imagegen.run_image_gen("a cat in the style of picasso", "1024x1024")) + llm = AzureLLMService() + tts = AzureTTSService() @transport.event_handler("on_participant_joined") - async def on_participant_joined(transport, participant): - (_, image_bytes) = await image_task - transport.output_queue.put(OutputQueueFrame(FrameType.IMAGE_FRAME, image_bytes)) + async def on_joined(transport, participant): + if participant["id"] == transport.my_participant_id: + return + + # queue two pieces of speech: one specified as a text literal, + # and one generated by an llm. We'll kick off the llm first, and let + # it generate a response while we're speaking the literal string. + llm_response_task = asyncio.create_task(llm.run_llm( + [{"role": "system", "content": "tell the user a joke about llamas"}] + )) + + async for audio_chunk in tts.run_tts("My friend the LLM is now going to tell a joke about llamas."): + transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio_chunk)) + + llm_response = await llm_response_task + async for audio_chunk in tts.run_tts(llm_response): + transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio_chunk)) + + + # wait for the output queue to be empty, then leave the meeting + transport.output_queue.join() + transport.stop() await transport.run() diff --git a/src/samples/theoretical-to-real/05-queued.py b/src/samples/theoretical-to-real/05-queued.py new file mode 100644 index 000000000..aa6da0257 --- /dev/null +++ b/src/samples/theoretical-to-real/05-queued.py @@ -0,0 +1,119 @@ +import argparse +import asyncio + +from asyncio.queues import Queue +import re + +from dailyai.output_queue import OutputQueueFrame, FrameType +from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from dailyai.services.open_ai_services import OpenAILLMService, OpenAIImageGenService +from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from dailyai.services.daily_transport_service import DailyTransportService + +async def main(room_url): + meeting_duration_minutes = 5 + transport = DailyTransportService( + room_url, + None, + "Month Narration Bot", + meeting_duration_minutes, + ) + transport.mic_enabled = True + transport.camera_enabled = True + transport.mic_sample_rate = 16000 + transport.camera_width = 1024 + transport.camera_height = 1024 + + llm = AzureLLMService() + tts = ElevenLabsTTSService() + dalle = OpenAIImageGenService() + + # Get a complete audio chunk from the given text. Splitting this into its own + # coroutine lets us ensure proper ordering of the audio chunks on the output queue. + async def get_all_audio(text): + all_audio = bytearray() + async for audio in tts.run_tts(text): + all_audio.extend(audio) + + return all_audio + + async def get_month_data(month): + image_text = "" + current_clause = "" + tts_tasks = [] + async for text in llm.run_llm_async( + [ + { + "role": "system", + "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please." + } + ] + ): + image_text += text + current_clause += text + if re.match(r"^.*[.!?]$", text): + tts_tasks.append(get_all_audio(current_clause)) + current_clause = "" + + tts_tasks.insert(0, dalle.run_image_gen(image_text, "1024x1024")) + + data = await asyncio.gather( + *tts_tasks + ) + + return { + "month": month, + "text": image_text, + "image": data[0][1], + "audio": data[1:], + } + + months: list[str] = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + ] + + @transport.event_handler("on_participant_joined") + async def on_participant_joined(transport, participant): + if participant["id"] == transport.my_participant_id: + return + + for month_data_task in asyncio.as_completed(month_tasks): + data = await month_data_task + transport.output_queue.put( + [ + OutputQueueFrame(FrameType.IMAGE_FRAME, data["image"]), + OutputQueueFrame(FrameType.AUDIO_FRAME, data["audio"][0]), + ] + ) + for audio in data["audio"][1:]: + transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio)) + + # wait for the output queue to be empty, then leave the meeting + transport.output_queue.join() + transport.stop() + + month_tasks = [asyncio.create_task(get_month_data(month)) for month in months] + + await transport.run() + +if __name__=="__main__": + parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=True, help="URL of the Daily room to join" + ) + + args: argparse.Namespace = parser.parse_args() + + asyncio.run(main(args.url)) diff --git a/src/samples/theoretical-to-real/05-sync-speech-and-text.py b/src/samples/theoretical-to-real/05-sync-speech-and-text.py index 111fb830e..e03fc15c6 100644 --- a/src/samples/theoretical-to-real/05-sync-speech-and-text.py +++ b/src/samples/theoretical-to-real/05-sync-speech-and-text.py @@ -1,4 +1,5 @@ import argparse + import asyncio from asyncio.queues import Queue