From 290c1e7efa188823cc753fb47320839882826fea Mon Sep 17 00:00:00 2001 From: chadbailey59 Date: Mon, 8 Jan 2024 16:07:21 -0600 Subject: [PATCH] Added async OpenAI services (#1) * added async openai services * added async openai services * added Deepgram service with 05 example * modernized the 'say one thing' example * async all the things * cleanup and user greeting * more cleanup --- .gitignore | 4 +- README.md | 9 +++ src/dailyai/services/deepgram_ai_services.py | 29 ++++++++++ ...open_ai_service.py => open_ai_services.py} | 45 ++++++++------- src/samples/simple-sample/simple-sample.py | 1 + .../theoretical-to-real/01a-greet-user.py | 55 +++++++++++++++++++ .../05-sync-speech-and-text.py | 14 +++-- 7 files changed, 130 insertions(+), 27 deletions(-) create mode 100644 src/dailyai/services/deepgram_ai_services.py rename src/dailyai/services/{open_ai_service.py => open_ai_services.py} (55%) create mode 100644 src/samples/theoretical-to-real/01a-greet-user.py diff --git a/.gitignore b/.gitignore index 85fe741a6..62f24b6da 100644 --- a/.gitignore +++ b/.gitignore @@ -23,5 +23,5 @@ share/python-wheels/ .installed.cfg *.egg MANIFEST - -.DS_Store \ No newline at end of file +.DS_Store +.env diff --git a/README.md b/README.md index ccd2a3a85..e16f073fd 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,14 @@ This SDK can help you build applications that participate in WebRTC meetings and ## Build/Install _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_ + ``` python3 -m venv env source env/bin/activate ``` From the root of this repo, run the following: + ``` pip install -r requirements.txt python -m build @@ -23,6 +25,7 @@ pip install . ``` If you want to use this package from another directory, you can run: + ``` pip install path_to_this_repo ``` @@ -44,3 +47,9 @@ AZURE_CHATGPT_KEY AZURE_CHATGPT_ENDPOINT AZURE_CHATGPT_DEPLOYMENT_ID ``` + +If you have those environment variables stored in an .env file, you can quickly load them into your terminal's environment by running this: + +```bash +$ export $(grep -v '^#' .env | xargs) +``` diff --git a/src/dailyai/services/deepgram_ai_services.py b/src/dailyai/services/deepgram_ai_services.py new file mode 100644 index 000000000..9daec8d3a --- /dev/null +++ b/src/dailyai/services/deepgram_ai_services.py @@ -0,0 +1,29 @@ +import aiohttp +import asyncio +import os + +import requests + +from collections.abc import AsyncGenerator +from dailyai.services.ai_services import TTSService + +class DeepgramTTSService(TTSService): + def __init__(self, speech_key=None, voice=None): + super().__init__() + + self.voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2" + self.speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY") + + def get_mic_sample_rate(self): + return 24000 + + async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]: + self.logger.info(f"Running deepgram tts for {sentence}") + base_url = "https://api.beta.deepgram.com/v1/speak" + request_url = f"{base_url}?model={self.voice}&encoding=linear16&container=none&sample_rate=16000" + headers = {"authorization": f"token {self.speech_key}"} + body = { "text": sentence } + async with aiohttp.ClientSession() as session: + async with session.post(request_url, headers=headers, json=body) as r: + async for data in r.content: + yield data \ No newline at end of file diff --git a/src/dailyai/services/open_ai_service.py b/src/dailyai/services/open_ai_services.py similarity index 55% rename from src/dailyai/services/open_ai_service.py rename to src/dailyai/services/open_ai_services.py index 64f650de7..9d17a7c87 100644 --- a/src/dailyai/services/open_ai_service.py +++ b/src/dailyai/services/open_ai_services.py @@ -1,33 +1,36 @@ -from dailyai.services.ai_services import AIService, TTSService, LLMService, ImageGenService -from typing import Generator - import requests +import aiohttp +import asyncio from PIL import Image import io -from openai import OpenAI +from openai import AsyncOpenAI import os import json +from collections.abc import AsyncGenerator + +from dailyai.services.ai_services import AIService, TTSService, LLMService, ImageGenService + class OpenAILLMService(LLMService): def __init__(self, api_key=None, model=None): super().__init__() api_key = api_key or os.getenv("OPEN_AI_KEY") - self.model = model or os.getenv("OPEN_AI_MODEL") - self.client = OpenAI(api_key=api_key) + self.model = model or os.getenv("OPEN_AI_LLM_MODEL") or "gpt-4" + self.client = AsyncOpenAI(api_key=api_key) - def get_response(self, messages, stream): - return self.client.chat.completions.create( + async def get_response(self, messages, stream): + return await self.client.chat.completions.create( stream=stream, messages=messages, model=self.model ) - def run_llm_async(self, messages) -> Generator[str, None, None]: + async def run_llm_async(self, messages) -> AsyncGenerator[str, None]: messages_for_log = json.dumps(messages) self.logger.debug(f"Generating chat via openai: {messages_for_log}") - response = self.get_response(messages, stream=True) + response = await self.get_response(messages, stream=True) for chunk in response: if len(chunk.choices) == 0: @@ -36,11 +39,11 @@ class OpenAILLMService(LLMService): if chunk.choices[0].delta.content: yield chunk.choices[0].delta.content - def run_llm(self, messages) -> str | None: + async def run_llm(self, messages) -> str | None: messages_for_log = json.dumps(messages) - self.logger.debug(f"Generating chat via azure: {messages_for_log}") + self.logger.debug(f"Generating chat via openai: {messages_for_log}") - response = self.get_response(messages, stream=False) + response = await self.get_response(messages, stream=False) if response and len(response.choices) > 0: return response.choices[0].message.content else: @@ -50,18 +53,22 @@ class OpenAIImageGenService(ImageGenService): def __init__(self, api_key=None, model=None): super().__init__() api_key = api_key or os.getenv("OPEN_AI_KEY") - self.model = model or os.getenv("OPEN_AI_MODEL") - self.client = OpenAI(api_key=api_key) + self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3" + self.client = AsyncOpenAI(api_key=api_key) - def run_image_gen(self, sentence) -> tuple[str, Image.Image]: - image = self.client.images.generate( + async def run_image_gen(self, sentence, size) -> tuple[str, bytes]: + self.logger.info("Generating OpenAI image", sentence) + + image = await self.client.images.generate( prompt=sentence, + model=self.model, n=1, - size=f"1024x1024" + size=size ) image_url = image.data[0].url response = requests.get(image_url) + dalle_stream = io.BytesIO(response.content) dalle_im = Image.open(dalle_stream) - return (image_url, dalle_im) + return (image_url, dalle_im.tobytes()) diff --git a/src/samples/simple-sample/simple-sample.py b/src/samples/simple-sample/simple-sample.py index 324c21603..27ae3fece 100644 --- a/src/samples/simple-sample/simple-sample.py +++ b/src/samples/simple-sample/simple-sample.py @@ -13,6 +13,7 @@ from dailyai.orchestrator import OrchestratorConfig, Orchestrator from dailyai.message_handler.message_handler import MessageHandler from dailyai.services.ai_services import AIServiceConfig from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService +from dailyai.services.deepgram_ai_services import DeepgramTTSService def add_bot_to_room(room_url, token, expiration) -> None: diff --git a/src/samples/theoretical-to-real/01a-greet-user.py b/src/samples/theoretical-to-real/01a-greet-user.py new file mode 100644 index 000000000..d3edb9dd0 --- /dev/null +++ b/src/samples/theoretical-to-real/01a-greet-user.py @@ -0,0 +1,55 @@ +import asyncio +import time +from typing import AsyncGenerator + +from dailyai.output_queue import OutputQueueFrame, FrameType +from dailyai.services.daily_transport_service import DailyTransportService +from dailyai.services.azure_ai_services import AzureTTSService +from dailyai.services.deepgram_ai_services import DeepgramTTSService + +async def main(room_url): + # create a transport service object using environment variables for + # the transport service's API key, room url, and any other configuration. + # services can all define and document the environment variables they use. + # services all also take an optional config object that is used instead of + # environment variables. + # + # the abstract transport service APIs presumably can map pretty closely + # to the daily-python basic API + meeting_duration_minutes = 1 + transport = DailyTransportService( + room_url, + None, + "Greeter", + meeting_duration_minutes, + ) + transport.mic_enabled = True + + # similarly, create a tts service + tts = DeepgramTTSService() + + # Get the generator for the audio. This will start running in the background, + # and when we ask the generator for its items, we'll get what it's generated. + + # Register an event handler so we can play the audio when the participant joins. + print("settting up handler") + @transport.event_handler("on_participant_joined") + async def on_participant_joined(transport, participant): + print(f"participant joined: {participant['info']['userName']}") + if participant["info"]["isLocal"]: + return + audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(f"Hello there, {participant['info']['userName']}!") + + async for audio in audio_generator: + transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio)) + + print("setting up call state handler") + @transport.event_handler("on_call_state_updated") + async def on_call_joined(transport, state): + print(f"call state callback: {state}") + + await transport.run() + + +if __name__ == "__main__": + asyncio.run(main("https://chad-hq.daily.co/howdy")) diff --git a/src/samples/theoretical-to-real/05-sync-speech-and-text.py b/src/samples/theoretical-to-real/05-sync-speech-and-text.py index 26a1e3db4..eedbb6b02 100644 --- a/src/samples/theoretical-to-real/05-sync-speech-and-text.py +++ b/src/samples/theoretical-to-real/05-sync-speech-and-text.py @@ -1,7 +1,9 @@ import asyncio from dailyai.output_queue import OutputQueueFrame, FrameType -from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService, AzureImageGenServiceREST +from dailyai.services.azure_ai_services import AzureTTSService +from dailyai.services.open_ai_services import OpenAILLMService, OpenAIImageGenService +from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.daily_transport_service import DailyTransportService async def main(room_url, token): @@ -23,9 +25,9 @@ async def main(room_url, token): transport.camera_width = 1024 transport.camera_height = 1024 - llm = AzureLLMService() - tts = AzureTTSService() - dalle = AzureImageGenServiceREST() + llm = OpenAILLMService() + tts = DeepgramTTSService() + dalle = OpenAIImageGenService() async def get_all_audio(text): all_audio = bytearray() @@ -44,7 +46,7 @@ async def main(room_url, token): } ] ) - print(f"got llm for {month}") + print(f"got llm for {month}, {inference_text}") (image, audio) = await asyncio.gather( *[dalle.run_image_gen(inference_text, "1024x1024"), get_all_audio(inference_text)] @@ -90,4 +92,4 @@ async def main(room_url, token): print("Done") if __name__=="__main__": - asyncio.run(main("https://moishe.daily.co/Lettvins", None)) + asyncio.run(main("https://chad-hq.daily.co/howdy", None))