From 72f631a066682171f941b9d6425919feb6b6b321 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 10 Mar 2024 17:21:46 -0700 Subject: [PATCH 1/6] working on foundational examples --- src/examples/foundational/01-say-one-thing.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/examples/foundational/01-say-one-thing.py b/src/examples/foundational/01-say-one-thing.py index 9f6a7eb34..388aabc6a 100644 --- a/src/examples/foundational/01-say-one-thing.py +++ b/src/examples/foundational/01-say-one-thing.py @@ -5,7 +5,6 @@ import os from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -from dailyai.services.playht_ai_service import PlayHTAIService from examples.support.runner import configure @@ -13,19 +12,14 @@ logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") logger = logging.getLogger("dailyai") logger.setLevel(logging.DEBUG) + async def main(room_url): async with aiohttp.ClientSession() as session: - # create a transport service object using environment variables for - # the transport service's API key, room url, and any other configuration. - # services can all define and document the environment variables they use. - # services all also take an optional config object that is used instead of - # environment variables. - # - # the abstract transport service APIs presumably can map pretty closely - # to the daily-python basic API - meeting_duration_minutes = 5 transport = DailyTransportService( - room_url, None, "Say One Thing", meeting_duration_minutes, mic_enabled=True + room_url, + None, + "Say One Thing", + mic_enabled=True, ) tts = ElevenLabsTTSService( @@ -37,7 +31,6 @@ async def main(room_url): # Register an event handler so we can play the audio when the participant joins. @transport.event_handler("on_participant_joined") async def on_participant_joined(transport, participant): - nonlocal tts if participant["info"]["isLocal"]: return From ef39d842a5f7587408ea571338a84c87a6a25b27 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 10 Mar 2024 19:18:37 -0700 Subject: [PATCH 2/6] custom processor in example 05 --- .../foundational/05-sync-speech-and-image.py | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/src/examples/foundational/05-sync-speech-and-image.py b/src/examples/foundational/05-sync-speech-and-image.py index f55cafcec..b7c76af6e 100644 --- a/src/examples/foundational/05-sync-speech-and-image.py +++ b/src/examples/foundational/05-sync-speech-and-image.py @@ -4,6 +4,9 @@ import aiohttp import os import logging +from dataclasses import dataclass +from typing import AsyncGenerator + from dailyai.pipeline.aggregators import ( GatedAggregator, LLMFullResponseAggregator, @@ -11,17 +14,20 @@ from dailyai.pipeline.aggregators import ( SentenceAggregator, ) from dailyai.pipeline.frames import ( - AudioFrame, + Frame, + TextFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, LLMResponseStartFrame, ) +from dailyai.pipeline.frame_processor import FrameProcessor + from dailyai.pipeline.pipeline import Pipeline -from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.open_ai_services import OpenAILLMService +from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from dailyai.services.fal_ai_services import FalImageGenService from examples.support.runner import configure @@ -30,14 +36,35 @@ logger = logging.getLogger("dailyai") logger.setLevel(logging.DEBUG) +@dataclass +class MonthFrame(Frame): + month: str + + +class MonthPrepender(FrameProcessor): + def __init__(self): + self.most_recent_month = "Placeholder, month frame not yet received" + self.prepend_to_next_text_frame = False + + async def process_frame(self, frame: Frame) -> AsyncGenerator[Frame, None]: + if isinstance(frame, MonthFrame): + self.most_recent_month = frame.month + elif self.prepend_to_next_text_frame and isinstance(frame, TextFrame): + yield TextFrame(f"{self.most_recent_month}: {frame.text}") + self.prepend_to_next_text_frame = False + elif isinstance(frame, LLMResponseStartFrame): + self.prepend_to_next_text_frame = True + yield frame + else: + yield frame + + async def main(room_url): async with aiohttp.ClientSession() as session: - meeting_duration_minutes = 5 transport = DailyTransportService( room_url, None, "Month Narration Bot", - duration_minutes=meeting_duration_minutes, mic_enabled=True, camera_enabled=True, mic_sample_rate=16000, @@ -55,8 +82,8 @@ async def main(room_url): api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview" ) - dalle = FalImageGenService( - image_size="1024x1024", + imagegen = FalImageGenService( + image_size="square_hd", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"), @@ -84,6 +111,7 @@ async def main(room_url): "content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.", } ] + await source_queue.put(MonthFrame(month)) await source_queue.put(LLMMessagesQueueFrame(messages)) await source_queue.put(EndFrame()) @@ -95,6 +123,7 @@ async def main(room_url): ) sentence_aggregator = SentenceAggregator() + month_prepender = MonthPrepender() llm_full_response_aggregator = LLMFullResponseAggregator() pipeline = Pipeline( @@ -103,7 +132,9 @@ async def main(room_url): processors=[ llm, sentence_aggregator, - ParallelPipeline([[tts], [llm_full_response_aggregator, dalle]]), + ParallelPipeline( + [[month_prepender, tts], [llm_full_response_aggregator, imagegen]] + ), gated_aggregator, ], ) @@ -112,8 +143,6 @@ async def main(room_url): @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): await pipeline_task - - # wait for the output queue to be empty, then leave the meeting await transport.stop_when_done() await transport.run() From 37e904ce6840846981bae39be18487fe95cd5551 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 10 Mar 2024 19:40:51 -0700 Subject: [PATCH 3/6] changed fal to a maybe slightly faster model --- src/dailyai/services/fal_ai_services.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/dailyai/services/fal_ai_services.py b/src/dailyai/services/fal_ai_services.py index 9464b46dd..4d7b0bb20 100644 --- a/src/dailyai/services/fal_ai_services.py +++ b/src/dailyai/services/fal_ai_services.py @@ -9,17 +9,19 @@ from dailyai.services.ai_services import ImageGenService from dailyai.services.ai_services import ImageGenService + # Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env class FalImageGenService(ImageGenService): def __init__( - self, - *, - image_size, - aiohttp_session: aiohttp.ClientSession, - key_id=None, - key_secret=None): + self, + *, + image_size, + aiohttp_session: aiohttp.ClientSession, + key_id=None, + key_secret=None + ): super().__init__(image_size) self._aiohttp_session = aiohttp_session if key_id: @@ -30,10 +32,9 @@ class FalImageGenService(ImageGenService): async def run_image_gen(self, sentence) -> tuple[str, bytes]: def get_image_url(sentence, size): handler = fal.apps.submit( - "110602490-fast-sdxl", - arguments={ - "prompt": sentence - }, + # "110602490-fast-sdxl", + "fal-ai/fast-sdxl", + arguments={"prompt": sentence}, ) for event in handler.iter_events(): if isinstance(event, fal.apps.InProgress): @@ -46,6 +47,7 @@ class FalImageGenService(ImageGenService): raise Exception("Image generation failed") return image_url + image_url = await asyncio.to_thread(get_image_url, sentence, self.image_size) # Load the image from the url async with self._aiohttp_session.get(image_url) as response: From 4396b1018a848abcf188e8b840fb7b985114f993 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 10 Mar 2024 19:41:32 -0700 Subject: [PATCH 4/6] small streamlining of example 02 --- .../foundational/02-llm-say-one-thing.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/examples/foundational/02-llm-say-one-thing.py b/src/examples/foundational/02-llm-say-one-thing.py index e1d182856..55a6863e8 100644 --- a/src/examples/foundational/02-llm-say-one-thing.py +++ b/src/examples/foundational/02-llm-say-one-thing.py @@ -6,24 +6,22 @@ import aiohttp from dailyai.pipeline.frames import LLMMessagesQueueFrame from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.open_ai_services import OpenAILLMService + from examples.support.runner import configure logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") logger = logging.getLogger("dailyai") logger.setLevel(logging.DEBUG) + async def main(room_url): async with aiohttp.ClientSession() as session: - meeting_duration_minutes = 1 transport = DailyTransportService( room_url, None, "Say One Thing From an LLM", - duration_minutes=meeting_duration_minutes, mic_enabled=True, ) @@ -32,25 +30,24 @@ async def main(room_url): api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"), ) + llm = OpenAILLMService( api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview" ) + messages = [ { "role": "system", "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.", } ] - tts_task = asyncio.create_task( - tts.run_to_queue( - transport.send_queue, - llm.run([LLMMessagesQueueFrame(messages)]), - ) - ) @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): - await tts_task + await tts.run_to_queue( + transport.send_queue, + llm.run([LLMMessagesQueueFrame(messages)]), + ) await transport.stop_when_done() await transport.run() From 959ffa9d366fa2252e7ff6c346c22d861106b732 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 10 Mar 2024 19:42:19 -0700 Subject: [PATCH 5/6] small streamlining of example 03 --- src/examples/foundational/03-still-frame.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/examples/foundational/03-still-frame.py b/src/examples/foundational/03-still-frame.py index 84b08f211..8099e97a0 100644 --- a/src/examples/foundational/03-still-frame.py +++ b/src/examples/foundational/03-still-frame.py @@ -6,48 +6,37 @@ import os from dailyai.pipeline.frames import TextFrame from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.fal_ai_services import FalImageGenService -from dailyai.services.open_ai_services import OpenAIImageGenService -from dailyai.services.azure_ai_services import AzureImageGenServiceREST from examples.support.runner import configure logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") logger = logging.getLogger("dailyai") logger.setLevel(logging.DEBUG) -local_joined = False -participant_joined = False async def main(room_url): async with aiohttp.ClientSession() as session: - meeting_duration_minutes = 1 transport = DailyTransportService( room_url, None, "Show a still frame image", - duration_minutes=meeting_duration_minutes, - mic_enabled=False, camera_enabled=True, camera_width=1024, camera_height=1024, ) imagegen = FalImageGenService( - image_size="1024x1024", + image_size="square_hd", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"), ) - image_task = asyncio.create_task( - imagegen.run_to_queue( - transport.send_queue, [TextFrame("a cat in the style of picasso")] - ) - ) - @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): - await image_task + await imagegen.run_to_queue( + transport.send_queue, [TextFrame("a cat in the style of picasso")] + ) await transport.run() From 4e16e514dd4d46fd77c80ab8bd3eb9b826336901 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Sun, 10 Mar 2024 19:43:06 -0700 Subject: [PATCH 6/6] attempting to change tts to deepgram in example 04 --- .../foundational/04-utterance-and-speech.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/examples/foundational/04-utterance-and-speech.py b/src/examples/foundational/04-utterance-and-speech.py index bcd2a6798..1d25a6c1a 100644 --- a/src/examples/foundational/04-utterance-and-speech.py +++ b/src/examples/foundational/04-utterance-and-speech.py @@ -7,6 +7,7 @@ from dailyai.pipeline.pipeline import Pipeline from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from examples.support.runner import configure @@ -25,7 +26,6 @@ async def main(room_url: str): duration_minutes=1, mic_enabled=True, mic_sample_rate=16000, - camera_enabled=False, ) llm = AzureLLMService( @@ -37,6 +37,11 @@ async def main(room_url: str): api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"), ) + + deepgram_tts = DeepgramTTSService( + aiohttp_session=session, + api_key=os.getenv("DEEPGRAM_API_KEY"), + ) elevenlabs_tts = ElevenLabsTTSService( aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), @@ -64,6 +69,15 @@ async def main(room_url: str): transport.send_queue, ) + # khk: deepgram_tts.say() doesn't seem to put bytes in the transport + # queue. I get a debug log line that indicates we're set up okay, but + # no further log lines or audio bytes. debug this later: + # 20 2024-03-10 13:24:46,235 Running deepgram tts for My friend the LLM is now going to tell a joke about llamas. + # await deepgram_tts.say( + # "My friend the LLM is now going to tell a joke about llamas.", + # transport.send_queue, + # ) + async def buffer_to_send_queue(): while True: frame = await buffer_queue.get() @@ -73,7 +87,6 @@ async def main(room_url: str): break await asyncio.gather(pipeline_run_task, buffer_to_send_queue()) - await transport.stop_when_done() await transport.run()