From ad427bea3a0677314c915d3d51abef53782c5cc3 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Fri, 12 Jan 2024 10:47:03 -0500 Subject: [PATCH] Update 05- sample, sticking with generators here because they give us a lot more control over the order that things get queued. --- src/dailyai/services/elevenlabs_ai_service.py | 2 +- src/dailyai/services/open_ai_services.py | 11 +++++-- .../04-utterance-and-speech.py | 4 +++ src/samples/theoretical-to-real/05-queued.py | 31 ++++++++++--------- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/dailyai/services/elevenlabs_ai_service.py b/src/dailyai/services/elevenlabs_ai_service.py index 482a98b67..0d9ec8b54 100644 --- a/src/dailyai/services/elevenlabs_ai_service.py +++ b/src/dailyai/services/elevenlabs_ai_service.py @@ -9,7 +9,7 @@ from dailyai.services.ai_services import TTSService class ElevenLabsTTSService(TTSService): - def __init__(self, input_queue, output_queue, api_key=None, voice_id=None): + def __init__(self, input_queue=None, output_queue=None, api_key=None, voice_id=None): super().__init__(input_queue, output_queue) self.api_key = api_key or os.getenv("ELEVENLABS_API_KEY") diff --git a/src/dailyai/services/open_ai_services.py b/src/dailyai/services/open_ai_services.py index 9d17a7c87..8f2b6154a 100644 --- a/src/dailyai/services/open_ai_services.py +++ b/src/dailyai/services/open_ai_services.py @@ -66,9 +66,14 @@ class OpenAIImageGenService(ImageGenService): size=size ) image_url = image.data[0].url - response = requests.get(image_url) + if not image_url: + raise Exception("No image provided in response", image) - dalle_stream = io.BytesIO(response.content) - dalle_im = Image.open(dalle_stream) + # Load the image from the url + async with aiohttp.ClientSession() as session: + async with session.get(image_url) as response: + image_stream = io.BytesIO(await response.content.read()) + image = Image.open(image_stream) + return (image_url, image.tobytes()) return (image_url, dalle_im.tobytes()) diff --git a/src/samples/theoretical-to-real/04-utterance-and-speech.py b/src/samples/theoretical-to-real/04-utterance-and-speech.py index fb5c479ad..0e4cb9caf 100644 --- a/src/samples/theoretical-to-real/04-utterance-and-speech.py +++ b/src/samples/theoretical-to-real/04-utterance-and-speech.py @@ -32,6 +32,10 @@ async def main(room_url:str): # queue two pieces of speech: one specified as a text literal, # and one generated by an llm. We'll kick off the llm first, and let # it generate a response while we're speaking the literal string. + # + # Note that in this case, we don't use `run_llm_async` because we're + # taking advantage of the time spent speaking the first phrase to generate + # the entire LLM response, and this happens asynchronously in a task. llm_response_task = asyncio.create_task(llm.run_llm( [{"role": "system", "content": "tell the user a joke about llamas"}] )) diff --git a/src/samples/theoretical-to-real/05-queued.py b/src/samples/theoretical-to-real/05-queued.py index 7958d5e00..60711f4bf 100644 --- a/src/samples/theoretical-to-real/05-queued.py +++ b/src/samples/theoretical-to-real/05-queued.py @@ -25,7 +25,7 @@ async def main(room_url): transport.camera_height = 1024 llm = AzureLLMService() - tts = ElevenLabsTTSService() + tts = ElevenLabsTTSService(voice_id="ErXwobaYiN019PkySvjV") dalle = OpenAIImageGenService() # Get a complete audio chunk from the given text. Splitting this into its own @@ -39,9 +39,9 @@ async def main(room_url): async def get_month_data(month): image_text = "" - current_clause = "" tts_tasks = [] - async for text in llm.run_llm_async( + first_sentence = True + async for sentence in llm.run_llm_async_sentences( [ { "role": "system", @@ -49,18 +49,24 @@ async def main(room_url): } ] ): - image_text += text - current_clause += text - if re.match(r"^.*[.!?]$", text): - tts_tasks.append(get_all_audio(current_clause)) - current_clause = "" + image_text += sentence + + if first_sentence: + sentence = f"{month}: {sentence}" + else: + first_sentence = False + + tts_tasks.append(get_all_audio(sentence)) tts_tasks.insert(0, dalle.run_image_gen(image_text, "1024x1024")) + print(f"waiting for tasks to finish for {month}") data = await asyncio.gather( *tts_tasks ) + print(f"done gathering tts tasks for {month}") + return { "month": month, "text": image_text, @@ -83,11 +89,8 @@ async def main(room_url): "December", ] - @transport.event_handler("on_participant_joined") - async def on_participant_joined(transport, participant): - if participant["id"] == transport.my_participant_id: - return - + @transport.event_handler("on_first_other_participant_joined") + async def on_first_other_participant_joined(transport): # This will play the months in the order they're completed. The benefit # is we'll have as little delay as possible before the first month, and # likely no delay between months, but the months won't display in order. @@ -116,6 +119,6 @@ if __name__=="__main__": "-u", "--url", type=str, required=True, help="URL of the Daily room to join" ) - args: argparse.Namespace = parser.parse_args() + args, unknown = parser.parse_known_args() asyncio.run(main(args.url))