Added async OpenAI services (#1)

* added async openai services

* added async openai services

* added Deepgram service with 05 example

* modernized the 'say one thing' example

* async all the things

* cleanup and user greeting

* more cleanup

.gitignore

@@ -23,5 +23,5 @@ share/python-wheels/
 .installed.cfg
 *.egg
 MANIFEST
-
-.DS_Store
+.DS_Store
+.env

README.md

@@ -5,12 +5,14 @@ This SDK can help you build applications that participate in WebRTC meetings and
 ## Build/Install
 
 _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_
+
 ```
 python3 -m venv env
 source env/bin/activate
 ```
 
 From the root of this repo, run the following:
+
 ```
 pip install -r requirements.txt
 python -m build
@@ -23,6 +25,7 @@ pip install .
 ```
 
 If you want to use this package from another directory, you can run:
+
 ```
 pip install path_to_this_repo
 ```
@@ -44,3 +47,9 @@ AZURE_CHATGPT_KEY
 AZURE_CHATGPT_ENDPOINT
 AZURE_CHATGPT_DEPLOYMENT_ID
 ```
+
+If you have those environment variables stored in an .env file, you can quickly load them into your terminal's environment by running this:
+
+```bash
+$ export $(grep -v '^#' .env | xargs)
+```

src/dailyai/services/deepgram_ai_services.py

@@ -0,0 +1,29 @@
+import aiohttp
+import asyncio
+import os
+
+import requests
+
+from collections.abc import AsyncGenerator
+from dailyai.services.ai_services import TTSService
+
+class DeepgramTTSService(TTSService):
+    def __init__(self, speech_key=None, voice=None):
+        super().__init__()
+
+        self.voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2"
+        self.speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY")
+    
+    def get_mic_sample_rate(self):
+        return 24000
+
+    async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
+        self.logger.info(f"Running deepgram tts for {sentence}")
+        base_url = "https://api.beta.deepgram.com/v1/speak"
+        request_url = f"{base_url}?model={self.voice}&encoding=linear16&container=none&sample_rate=16000"
+        headers = {"authorization": f"token {self.speech_key}"}
+        body = { "text": sentence }
+        async with aiohttp.ClientSession() as session:
+            async with session.post(request_url, headers=headers, json=body) as r:
+                async for data in r.content:
+                    yield data

src/dailyai/services/open_ai_services.py

@@ -1,33 +1,36 @@
-from dailyai.services.ai_services import AIService, TTSService, LLMService, ImageGenService
-from typing import Generator
-
 import requests
+import aiohttp
+import asyncio
 from PIL import Image
 import io
-from openai import OpenAI
+from openai import AsyncOpenAI
 
 import os
 import json
+from collections.abc import AsyncGenerator
+
+from dailyai.services.ai_services import AIService, TTSService, LLMService, ImageGenService
+
 
 class OpenAILLMService(LLMService):
     def __init__(self, api_key=None, model=None):
         super().__init__()
         api_key = api_key or os.getenv("OPEN_AI_KEY")
-        self.model = model or os.getenv("OPEN_AI_MODEL")
-        self.client = OpenAI(api_key=api_key)
+        self.model = model or os.getenv("OPEN_AI_LLM_MODEL") or "gpt-4"
+        self.client = AsyncOpenAI(api_key=api_key)
 
-    def get_response(self, messages, stream):
-        return self.client.chat.completions.create(
+    async def get_response(self, messages, stream):
+        return await self.client.chat.completions.create(
             stream=stream,
             messages=messages,
             model=self.model
         )
 
-    def run_llm_async(self, messages) -> Generator[str, None, None]:
+    async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
         messages_for_log = json.dumps(messages)
         self.logger.debug(f"Generating chat via openai: {messages_for_log}")
 
-        response = self.get_response(messages, stream=True)
+        response = await self.get_response(messages, stream=True)
 
         for chunk in response:
             if len(chunk.choices) == 0:
@@ -36,11 +39,11 @@ class OpenAILLMService(LLMService):
             if chunk.choices[0].delta.content:
                 yield chunk.choices[0].delta.content
 
-    def run_llm(self, messages) -> str | None:
+    async def run_llm(self, messages) -> str | None:
         messages_for_log = json.dumps(messages)
-        self.logger.debug(f"Generating chat via azure: {messages_for_log}")
+        self.logger.debug(f"Generating chat via openai: {messages_for_log}")
 
-        response = self.get_response(messages, stream=False)
+        response = await self.get_response(messages, stream=False)
         if response and len(response.choices) > 0:
             return response.choices[0].message.content
         else:
@@ -50,18 +53,22 @@ class OpenAIImageGenService(ImageGenService):
     def __init__(self, api_key=None, model=None):
         super().__init__()
         api_key = api_key or os.getenv("OPEN_AI_KEY")
-        self.model = model or os.getenv("OPEN_AI_MODEL")
-        self.client = OpenAI(api_key=api_key)
+        self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3"
+        self.client = AsyncOpenAI(api_key=api_key)
 
-    def run_image_gen(self, sentence) -> tuple[str, Image.Image]:
-        image = self.client.images.generate(
+    async def run_image_gen(self, sentence, size) -> tuple[str, bytes]:
+        self.logger.info("Generating OpenAI image", sentence)
+
+        image = await self.client.images.generate(
             prompt=sentence,
+            model=self.model,
             n=1,
-            size=f"1024x1024"
+            size=size
         )
         image_url = image.data[0].url
         response = requests.get(image_url)
+
         dalle_stream = io.BytesIO(response.content)
         dalle_im = Image.open(dalle_stream)
 
-        return (image_url, dalle_im)
+        return (image_url, dalle_im.tobytes())

src/samples/simple-sample/simple-sample.py

@@ -13,6 +13,7 @@ from dailyai.orchestrator import OrchestratorConfig, Orchestrator
 from dailyai.message_handler.message_handler import MessageHandler
 from dailyai.services.ai_services import AIServiceConfig
 from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
 
 def add_bot_to_room(room_url, token, expiration) -> None:
 

src/samples/theoretical-to-real/01a-greet-user.py

@@ -0,0 +1,55 @@
+import asyncio
+import time
+from typing import AsyncGenerator
+
+from dailyai.output_queue import OutputQueueFrame, FrameType
+from dailyai.services.daily_transport_service import DailyTransportService
+from dailyai.services.azure_ai_services import AzureTTSService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
+
+async def main(room_url):
+    # create a transport service object using environment variables for
+    # the transport service's API key, room url, and any other configuration.
+    # services can all define and document the environment variables they use.
+    # services all also take an optional config object that is used instead of
+    # environment variables.
+    #
+    # the abstract transport service APIs presumably can map pretty closely
+    # to the daily-python basic API
+    meeting_duration_minutes = 1
+    transport = DailyTransportService(
+        room_url,
+        None,
+        "Greeter",
+        meeting_duration_minutes,
+    )
+    transport.mic_enabled = True
+
+    # similarly, create a tts service
+    tts = DeepgramTTSService()
+
+    # Get the generator for the audio. This will start running in the background,
+    # and when we ask the generator for its items, we'll get what it's generated.
+
+    # Register an event handler so we can play the audio when the participant joins.
+    print("settting up handler")
+    @transport.event_handler("on_participant_joined")
+    async def on_participant_joined(transport, participant):
+        print(f"participant joined: {participant['info']['userName']}")
+        if participant["info"]["isLocal"]:
+            return
+        audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(f"Hello there, {participant['info']['userName']}!")
+
+        async for audio in audio_generator:
+            transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio))
+
+    print("setting up call state handler")
+    @transport.event_handler("on_call_state_updated")
+    async def on_call_joined(transport, state):
+        print(f"call state callback: {state}")
+    
+    await transport.run()
+
+
+if __name__ == "__main__":
+    asyncio.run(main("https://chad-hq.daily.co/howdy"))

src/samples/theoretical-to-real/05-sync-speech-and-text.py

@@ -1,7 +1,9 @@
 import asyncio
 
 from dailyai.output_queue import OutputQueueFrame, FrameType
-from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService, AzureImageGenServiceREST
+from dailyai.services.azure_ai_services import AzureTTSService
+from dailyai.services.open_ai_services import OpenAILLMService, OpenAIImageGenService
+from dailyai.services.deepgram_ai_services import DeepgramTTSService
 from dailyai.services.daily_transport_service import DailyTransportService
 
 async def main(room_url, token):
@@ -23,9 +25,9 @@ async def main(room_url, token):
     transport.camera_width = 1024
     transport.camera_height = 1024
 
-    llm = AzureLLMService()
-    tts = AzureTTSService()
-    dalle = AzureImageGenServiceREST()
+    llm = OpenAILLMService()
+    tts = DeepgramTTSService()
+    dalle = OpenAIImageGenService()
 
     async def get_all_audio(text):
         all_audio = bytearray()
@@ -44,7 +46,7 @@ async def main(room_url, token):
                 }
             ]
         )
-        print(f"got llm for {month}")
+        print(f"got llm for {month}, {inference_text}")
 
         (image, audio) = await asyncio.gather(
             *[dalle.run_image_gen(inference_text, "1024x1024"), get_all_audio(inference_text)]
@@ -90,4 +92,4 @@ async def main(room_url, token):
     print("Done")
 
 if __name__=="__main__":
-    asyncio.run(main("https://moishe.daily.co/Lettvins", None))
+    asyncio.run(main("https://chad-hq.daily.co/howdy", None))