From 70d07b6ea21f9edb9fc15a6a2c33a8abae3e4316 Mon Sep 17 00:00:00 2001 From: chadbailey59 Date: Tue, 6 Feb 2024 15:07:16 -0600 Subject: [PATCH] WIP: environment cleanup (#19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * removed env var usage from SDK services * started consolidating configure.py * 1–3 work * cleaned up the rest * more cleanup * cleanup and 05 tinkering * made fal keys optional --- pyproject.toml | 1 + src/dailyai/services/azure_ai_services.py | 56 +++--- src/dailyai/services/deepgram_ai_services.py | 8 +- src/dailyai/services/elevenlabs_ai_service.py | 9 +- src/dailyai/services/fal_ai_services.py | 8 +- src/dailyai/services/open_ai_services.py | 27 ++- src/samples/foundational/01-say-one-thing.py | 18 +- .../foundational/02-llm-say-one-thing.py | 25 +-- src/samples/foundational/03-still-frame.py | 21 +-- .../foundational/04-utterance-and-speech.py | 18 +- ...nd-text.py => 05-sync-speech-and-image.py} | 58 +++--- .../foundational/06-listen-and-respond.py | 45 +---- src/samples/foundational/06a-image-sync.py | 43 +---- src/samples/foundational/07-interruptible.py | 46 +---- src/samples/foundational/08-bots-arguing.py | 107 ++++++++---- .../foundational/08a-characters-arguing.py | 108 ------------ .../foundational/08b-debate-generator.py | 119 ------------- src/samples/foundational/10-wake-word.py | 56 ++---- src/samples/foundational/11-sound-effects.py | 139 ++++++--------- src/samples/foundational/11a-dial-out.py | 165 ------------------ .../foundational/13-whisper-transcription.py | 11 +- src/samples/foundational/support/runner.py | 52 ++++++ src/samples/internal/11a-dial-out.py | 135 ++++++++++++++ src/samples/server/daily-bot-manager.py | 4 +- 24 files changed, 475 insertions(+), 804 deletions(-) rename src/samples/foundational/{05-sync-speech-and-text.py => 05-sync-speech-and-image.py} (65%) delete mode 100644 src/samples/foundational/08a-characters-arguing.py delete mode 100644 src/samples/foundational/08b-debate-generator.py delete mode 100644 src/samples/foundational/11a-dial-out.py create mode 100644 src/samples/foundational/support/runner.py create mode 100644 src/samples/internal/11a-dial-out.py diff --git a/pyproject.toml b/pyproject.toml index 4f504960a..348652cfb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,6 +8,7 @@ version = "0.0.1" description = "Orchestrator for AI bots with Daily" dependencies = [ "daily-python", + "python-dotenv", "Pillow", "typing-extensions", "openai", diff --git a/src/dailyai/services/azure_ai_services.py b/src/dailyai/services/azure_ai_services.py index 64d542336..1f3e67cd2 100644 --- a/src/dailyai/services/azure_ai_services.py +++ b/src/dailyai/services/azure_ai_services.py @@ -17,13 +17,10 @@ from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, Resu class AzureTTSService(TTSService): - def __init__(self, speech_key=None, speech_region=None): + def __init__(self, *, api_key, region): super().__init__() - speech_key = speech_key or os.getenv("AZURE_SPEECH_SERVICE_KEY") - speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION") - - self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region) + self.speech_config = SpeechConfig(subscription=api_key, region=region) self.speech_synthesizer = SpeechSynthesizer( speech_config=self.speech_config, audio_config=None) @@ -51,25 +48,13 @@ class AzureTTSService(TTSService): class AzureLLMService(LLMService): - def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None): + def __init__(self, *, api_key, endpoint, api_version="2023-12-01-preview", model): super().__init__() - api_key = api_key or os.getenv("AZURE_CHATGPT_KEY") + self._model: str = model - azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT") - if not azure_endpoint: - raise Exception( - "No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor") - - model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID") - if not model: - raise Exception( - "No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor") - self.model: str = model - - api_version = api_version or "2023-12-01-preview" - self.client = AsyncAzureOpenAI( + self._client = AsyncAzureOpenAI( api_key=api_key, - azure_endpoint=azure_endpoint, + azure_endpoint=endpoint, api_version=api_version, ) @@ -77,7 +62,7 @@ class AzureLLMService(LLMService): messages_for_log = json.dumps(messages) self.logger.debug(f"Generating chat via azure: {messages_for_log}") - chunks = await self.client.chat.completions.create(model=self.model, stream=True, messages=messages) + chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages) async for chunk in chunks: if len(chunk.choices) == 0: continue @@ -89,7 +74,7 @@ class AzureLLMService(LLMService): messages_for_log = json.dumps(messages) self.logger.debug(f"Generating chat via azure: {messages_for_log}") - response = await self.client.chat.completions.create(model=self.model, stream=False, messages=messages) + response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages) if response and len(response.choices) > 0: return response.choices[0].message.content else: @@ -100,17 +85,19 @@ class AzureImageGenServiceREST(ImageGenService): def __init__( self, + *, + api_version="2023-06-01-preview", image_size: str, aiohttp_session: aiohttp.ClientSession, - api_key=None, - azure_endpoint=None, - api_version=None, - model=None): + api_key, + endpoint, + model): super().__init__(image_size=image_size) - self._api_key = api_key or os.getenv("AZURE_DALLE_KEY") - self._azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT") - self._api_version = api_version or "2023-06-01-preview" - self._model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID") + + self._api_key = api_key + self._azure_endpoint = endpoint + self._api_version = api_version + self._model = model self._aiohttp_session = aiohttp_session async def run_image_gen(self, sentence) -> tuple[str, bytes]: @@ -125,8 +112,11 @@ class AzureImageGenServiceREST(ImageGenService): async with self._aiohttp_session.post( url, headers=headers, json=body ) as submission: + print(f"submission: {submission}") + # We never get past this line, because this header isn't + # defined on a 429 response, but something is eating our exceptions! operation_location = submission.headers['operation-location'] - + print(f"submission status: {submission.status}") status = "" attempts_left = 120 json_response = None @@ -145,9 +135,9 @@ class AzureImageGenServiceREST(ImageGenService): image_url = json_response["result"]["data"][0]["url"] if json_response else None if not image_url: raise Exception("Image generation failed") - # Load the image from the url async with self._aiohttp_session.get(image_url) as response: image_stream = io.BytesIO(await response.content.read()) image = Image.open(image_stream) + print("i got an image file!") return (image_url, image.tobytes()) diff --git a/src/dailyai/services/deepgram_ai_services.py b/src/dailyai/services/deepgram_ai_services.py index 781e1dd88..ff6563023 100644 --- a/src/dailyai/services/deepgram_ai_services.py +++ b/src/dailyai/services/deepgram_ai_services.py @@ -9,11 +9,11 @@ from dailyai.services.ai_services import TTSService class DeepgramTTSService(TTSService): - def __init__(self, aiohttp_session, speech_key=None, voice=None): + def __init__(self, *, aiohttp_session, api_key, voice="alpha-asteria-en-v2"): super().__init__() - self._voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2" - self._speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY") + self._voice = voice + self._api_key = api_key self._aiohttp_session = aiohttp_session def get_mic_sample_rate(self): @@ -23,7 +23,7 @@ class DeepgramTTSService(TTSService): self.logger.info(f"Running deepgram tts for {sentence}") base_url = "https://api.beta.deepgram.com/v1/speak" request_url = f"{base_url}?model={self._voice}&encoding=linear16&container=none&sample_rate=16000" - headers = {"authorization": f"token {self._speech_key}"} + headers = {"authorization": f"token {self._api_key}"} body = {"text": sentence} async with self._aiohttp_session.post(request_url, headers=headers, json=body) as r: async for data in r.content: diff --git a/src/dailyai/services/elevenlabs_ai_service.py b/src/dailyai/services/elevenlabs_ai_service.py index e5951d933..e1795aab3 100644 --- a/src/dailyai/services/elevenlabs_ai_service.py +++ b/src/dailyai/services/elevenlabs_ai_service.py @@ -12,14 +12,15 @@ class ElevenLabsTTSService(TTSService): def __init__( self, + *, aiohttp_session: aiohttp.ClientSession, - api_key=None, - voice_id=None, + api_key, + voice_id, ): super().__init__() - self._api_key = api_key or os.getenv("ELEVENLABS_API_KEY") - self._voice_id = voice_id or os.getenv("ELEVENLABS_VOICE_ID") + self._api_key = api_key + self._voice_id = voice_id self._aiohttp_session = aiohttp_session async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]: diff --git a/src/dailyai/services/fal_ai_services.py b/src/dailyai/services/fal_ai_services.py index ef9f8a801..2f56f3be7 100644 --- a/src/dailyai/services/fal_ai_services.py +++ b/src/dailyai/services/fal_ai_services.py @@ -2,18 +2,22 @@ import fal import aiohttp import asyncio import io +import os import json from PIL import Image from dailyai.services.ai_services import LLMService, TTSService, ImageGenService -# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env class FalImageGenService(ImageGenService): - def __init__(self, image_size, aiohttp_session: aiohttp.ClientSession): + def __init__(self, *, image_size, aiohttp_session: aiohttp.ClientSession, key_id=None, key_secret=None): super().__init__(image_size) self._aiohttp_session = aiohttp_session + if key_id: + os.environ["FAL_KEY_ID"] = key_id + if key_secret: + os.environ["FAL_KEY_SECRET"] = key_secret async def run_image_gen(self, sentence) -> tuple[str, bytes]: def get_image_url(sentence, size): diff --git a/src/dailyai/services/open_ai_services.py b/src/dailyai/services/open_ai_services.py index 072f64022..7892af06b 100644 --- a/src/dailyai/services/open_ai_services.py +++ b/src/dailyai/services/open_ai_services.py @@ -13,26 +13,24 @@ from dailyai.services.ai_services import AIService, TTSService, LLMService, Imag class OpenAILLMService(LLMService): - def __init__(self, api_key=None, model=None): + def __init__(self, *, api_key, model="gpt-4"): super().__init__() - api_key = api_key or os.getenv("OPEN_AI_KEY") - self.model = model or os.getenv("OPEN_AI_LLM_MODEL") or "gpt-4" - self.client = AsyncOpenAI(api_key=api_key) + self._model = model + self._client = AsyncOpenAI(api_key=api_key) async def get_response(self, messages, stream): - return await self.client.chat.completions.create( + return await self._client.chat.completions.create( stream=stream, messages=messages, - model=self.model + model=self._model ) async def run_llm_async(self, messages) -> AsyncGenerator[str, None]: messages_for_log = json.dumps(messages) self.logger.debug(f"Generating chat via openai: {messages_for_log}") - response = await self.get_response(messages, stream=True) - - for chunk in response: + chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages) + async for chunk in chunks: if len(chunk.choices) == 0: continue @@ -43,7 +41,7 @@ class OpenAILLMService(LLMService): messages_for_log = json.dumps(messages) self.logger.debug(f"Generating chat via openai: {messages_for_log}") - response = await self.get_response(messages, stream=False) + response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages) if response and len(response.choices) > 0: return response.choices[0].message.content else: @@ -54,14 +52,15 @@ class OpenAIImageGenService(ImageGenService): def __init__( self, + *, image_size: str, aiohttp_session: aiohttp.ClientSession, - api_key=None, - model=None, + api_key, + model="dall-e-3", ): super().__init__(image_size=image_size) - api_key = api_key or os.getenv("OPEN_AI_KEY") - self._model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3" + self._model = model + print(f"api key: {api_key}") self._client = AsyncOpenAI(api_key=api_key) self._aiohttp_session = aiohttp_session diff --git a/src/samples/foundational/01-say-one-thing.py b/src/samples/foundational/01-say-one-thing.py index 780a2ce45..39a492fae 100644 --- a/src/samples/foundational/01-say-one-thing.py +++ b/src/samples/foundational/01-say-one-thing.py @@ -1,11 +1,11 @@ -import argparse import asyncio - import aiohttp +import os from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from samples.foundational.support.runner import configure async def main(room_url): async with aiohttp.ClientSession() as session: @@ -17,7 +17,7 @@ async def main(room_url): # # the abstract transport service APIs presumably can map pretty closely # to the daily-python basic API - meeting_duration_minutes = 1 + meeting_duration_minutes = 5 transport = DailyTransportService( room_url, None, @@ -25,7 +25,7 @@ async def main(room_url): meeting_duration_minutes, ) transport.mic_enabled = True - tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV") + tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) # Register an event handler so we can play the audio when the participant joins. @transport.event_handler("on_participant_joined") @@ -45,11 +45,5 @@ async def main(room_url): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - - asyncio.run(main(args.url)) + (url, token) = configure() + asyncio.run(main(url)) diff --git a/src/samples/foundational/02-llm-say-one-thing.py b/src/samples/foundational/02-llm-say-one-thing.py index 05c72e687..d182ebb7b 100644 --- a/src/samples/foundational/02-llm-say-one-thing.py +++ b/src/samples/foundational/02-llm-say-one-thing.py @@ -1,13 +1,16 @@ import argparse import asyncio +import os import aiohttp from dailyai.queue_frame import LLMMessagesQueueFrame from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.azure_ai_services import AzureLLMService +from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService - +from dailyai.services.deepgram_ai_services import DeepgramTTSService +from dailyai.services.open_ai_services import OpenAILLMService +from samples.foundational.support.runner import configure async def main(room_url): async with aiohttp.ClientSession() as session: @@ -20,9 +23,12 @@ async def main(room_url): ) transport.mic_enabled = True - tts = ElevenLabsTTSService(session, voice_id="29vD33N1CtxCmqQRPOHJ") - llm = AzureLLMService() - + tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + # tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE")) + + # llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY")) messages = [{ "role": "system", "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world." @@ -43,10 +49,5 @@ async def main(room_url): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - asyncio.run(main(args.url)) + (url, token) = configure() + asyncio.run(main(url)) diff --git a/src/samples/foundational/03-still-frame.py b/src/samples/foundational/03-still-frame.py index 6aa390095..d5cb2124b 100644 --- a/src/samples/foundational/03-still-frame.py +++ b/src/samples/foundational/03-still-frame.py @@ -1,11 +1,15 @@ import argparse import asyncio - import aiohttp +import os from dailyai.queue_frame import TextQueueFrame from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.fal_ai_services import FalImageGenService +from dailyai.services.open_ai_services import OpenAIImageGenService +from dailyai.services.azure_ai_services import AzureImageGenServiceREST + +from samples.foundational.support.runner import configure local_joined = False participant_joined = False @@ -25,7 +29,10 @@ async def main(room_url): transport.camera_width = 1024 transport.camera_height = 1024 - imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session) + imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + # imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024") + # imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL")) + image_task = asyncio.create_task( imagegen.run_to_queue( transport.send_queue, [ @@ -39,11 +46,5 @@ async def main(room_url): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - - asyncio.run(main(args.url)) + (url, token) = configure() + asyncio.run(main(url)) diff --git a/src/samples/foundational/04-utterance-and-speech.py b/src/samples/foundational/04-utterance-and-speech.py index 8e6b8ff47..968d9d896 100644 --- a/src/samples/foundational/04-utterance-and-speech.py +++ b/src/samples/foundational/04-utterance-and-speech.py @@ -1,5 +1,6 @@ import argparse import asyncio +import os import re import aiohttp @@ -9,6 +10,7 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from samples.foundational.support.runner import configure async def main(room_url: str): async with aiohttp.ClientSession() as session: @@ -22,9 +24,9 @@ async def main(room_url: str): transport.mic_sample_rate = 16000 transport.camera_enabled = False - llm = AzureLLMService() - azure_tts = AzureTTSService() - elevenlabs_tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV") + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + azure_tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + elevenlabs_tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) messages = [{"role": "system", "content": "tell the user a joke about llamas"}] @@ -60,11 +62,5 @@ async def main(room_url: str): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - - asyncio.run(main(args.url)) + (url, token) = configure() + asyncio.run(main(url)) diff --git a/src/samples/foundational/05-sync-speech-and-text.py b/src/samples/foundational/05-sync-speech-and-image.py similarity index 65% rename from src/samples/foundational/05-sync-speech-and-text.py rename to src/samples/foundational/05-sync-speech-and-image.py index 3bc9be606..77e95fc7b 100644 --- a/src/samples/foundational/05-sync-speech-and-text.py +++ b/src/samples/foundational/05-sync-speech-and-image.py @@ -1,14 +1,16 @@ import argparse import asyncio - import aiohttp +import os from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame -from dailyai.services.azure_ai_services import AzureLLMService +from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.fal_ai_services import FalImageGenService +from dailyai.services.open_ai_services import OpenAIImageGenService +from samples.foundational.support.runner import configure async def main(room_url): async with aiohttp.ClientSession() as session: @@ -25,11 +27,14 @@ async def main(room_url): transport.camera_width = 1024 transport.camera_height = 1024 - llm = AzureLLMService() - dalle = FalImageGenService(aiohttp_session=session, image_size="1024x1024") - tts = ElevenLabsTTSService(aiohttp_session=session, voice_id="ErXwobaYiN019PkySvjV") - # dalle = OpenAIImageGenService(image_size="1024x1024") + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV") + # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + # dalle = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024") + # dalle = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL")) + # Get a complete audio chunk from the given text. Splitting this into its own # coroutine lets us ensure proper ordering of the audio chunks on the send queue. async def get_all_audio(text): @@ -54,10 +59,11 @@ async def main(room_url): to_speak = f"{month}: {image_description}" audio_task = asyncio.create_task(get_all_audio(to_speak)) image_task = asyncio.create_task(dalle.run_image_gen(image_description)) + print(f"about to gather tasks for {month}") (audio, image_data) = await asyncio.gather( audio_task, image_task ) - + print(f"about to return from get_month_data for {month}") return { "month": month, "text": image_description, @@ -72,22 +78,32 @@ async def main(room_url): "March", "April", "May", - "June", - "July", - "August", - "September", - "October", - "November", - "December", + "June" ] - + """ + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", + """ @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): # This will play the months in the order they're completed. The benefit # is we'll have as little delay as possible before the first month, and # likely no delay between months, but the months won't display in order. for month_data_task in asyncio.as_completed(month_tasks): - data = await month_data_task + print(f"month_data_task: {month_data_task}") + try: + data = await month_data_task + except Exception: + print("OMG EXCEPTION!!!!") if data: await transport.send_queue.put( [ @@ -104,11 +120,5 @@ async def main(room_url): await transport.run() if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - - asyncio.run(main(args.url)) + (url, token) = configure() + asyncio.run(main(url)) \ No newline at end of file diff --git a/src/samples/foundational/06-listen-and-respond.py b/src/samples/foundational/06-listen-and-respond.py index 438c6e0ca..d10ff1e0c 100644 --- a/src/samples/foundational/06-listen-and-respond.py +++ b/src/samples/foundational/06-listen-and-respond.py @@ -1,13 +1,10 @@ -import argparse import asyncio -import requests -import time -import urllib.parse +import os from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator - +from samples.foundational.support.runner import configure async def main(room_url: str, token): global transport @@ -24,8 +21,8 @@ async def main(room_url: str, token): transport.mic_sample_rate = 16000 transport.camera_enabled = False - llm = AzureLLMService() - tts = AzureTTSService() + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): @@ -54,35 +51,5 @@ async def main(room_url: str, token): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - parser.add_argument( - "-k", - "--apikey", - type=str, - required=True, - help="Daily API Key (needed to create token)", - ) - - args, unknown = parser.parse_known_args() - - # Create a meeting token for the given room with an expiration 1 hour in the future. - room_name: str = urllib.parse.urlparse(args.url).path[1:] - expiration: float = time.time() + 60 * 60 - - res: requests.Response = requests.post( - f"https://api.daily.co/v1/meeting-tokens", - headers={"Authorization": f"Bearer {args.apikey}"}, - json={ - "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} - }, - ) - - if res.status_code != 200: - raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") - - token: str = res.json()["token"] - - asyncio.run(main(args.url, token)) + (url, token) = configure() + asyncio.run(main(url, token)) diff --git a/src/samples/foundational/06a-image-sync.py b/src/samples/foundational/06a-image-sync.py index f8898ebe4..2c280044b 100644 --- a/src/samples/foundational/06a-image-sync.py +++ b/src/samples/foundational/06a-image-sync.py @@ -16,6 +16,7 @@ from dailyai.services.ai_services import AIService from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator from dailyai.services.fal_ai_services import FalImageGenService +from samples.foundational.support.runner import configure class ImageSyncAggregator(AIService): def __init__(self, speaking_path: str, waiting_path: str): @@ -32,7 +33,7 @@ class ImageSyncAggregator(AIService): async def main(room_url: str, token): - async with aiohttp.ClientSession() as aiohttp_session: + async with aiohttp.ClientSession() as session: transport = DailyTransportService( room_url, token, @@ -45,9 +46,9 @@ async def main(room_url: str, token): transport.mic_enabled = True transport.mic_sample_rate = 16000 - llm = AzureLLMService() - tts = AzureTTSService() - img = FalImageGenService(image_size="1024x1024", aiohttp_session=aiohttp_session) + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + img = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) async def get_images(): get_speaking_task = asyncio.create_task( @@ -100,35 +101,5 @@ async def main(room_url: str, token): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - parser.add_argument( - "-k", - "--apikey", - type=str, - required=True, - help="Daily API Key (needed to create token)", - ) - - args, unknown = parser.parse_known_args() - - # Create a meeting token for the given room with an expiration 1 hour in the future. - room_name: str = urllib.parse.urlparse(args.url).path[1:] - expiration: float = time.time() + 60 * 60 - - res: requests.Response = requests.post( - f"https://api.daily.co/v1/meeting-tokens", - headers={"Authorization": f"Bearer {args.apikey}"}, - json={ - "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} - }, - ) - - if res.status_code != 200: - raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") - - token: str = res.json()["token"] - - asyncio.run(main(args.url, token)) + (url, token) = configure() + asyncio.run(main(url, token)) diff --git a/src/samples/foundational/07-interruptible.py b/src/samples/foundational/07-interruptible.py index 9e84c0a5a..532cdf11d 100644 --- a/src/samples/foundational/07-interruptible.py +++ b/src/samples/foundational/07-interruptible.py @@ -1,16 +1,14 @@ -import argparse import asyncio import aiohttp -import requests -import time -import urllib.parse +import os from dailyai.conversation_wrappers import InterruptibleConversationWrapper from dailyai.queue_frame import StartStreamQueueFrame, TextQueueFrame from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.azure_ai_services import AzureLLMService +from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from samples.foundational.support.runner import configure async def main(room_url: str, token): async with aiohttp.ClientSession() as session: @@ -25,8 +23,8 @@ async def main(room_url: str, token): transport.camera_enabled = False transport.start_transcription = True - llm = AzureLLMService() - tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV") + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) async def run_response(user_speech, tma_in, tma_out): await tts.run_to_queue( @@ -63,35 +61,5 @@ async def main(room_url: str, token): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - parser.add_argument( - "-k", - "--apikey", - type=str, - required=True, - help="Daily API Key (needed to create token)", - ) - - args, unknown = parser.parse_known_args() - - # Create a meeting token for the given room with an expiration 1 hour in the future. - room_name: str = urllib.parse.urlparse(args.url).path[1:] - expiration: float = time.time() + 60 * 60 - - res: requests.Response = requests.post( - f"https://api.daily.co/v1/meeting-tokens", - headers={"Authorization": f"Bearer {args.apikey}"}, - json={ - "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} - }, - ) - - if res.status_code != 200: - raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") - - token: str = res.json()["token"] - - asyncio.run(main(args.url, token)) + (url, token) = configure() + asyncio.run(main(url, token)) diff --git a/src/samples/foundational/08-bots-arguing.py b/src/samples/foundational/08-bots-arguing.py index ef1384260..00f7bc64f 100644 --- a/src/samples/foundational/08-bots-arguing.py +++ b/src/samples/foundational/08-bots-arguing.py @@ -1,10 +1,14 @@ import aiohttp -import argparse import asyncio +import os from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from dailyai.services.fal_ai_services import FalImageGenService +from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame + +from samples.foundational.support.runner import configure async def main(room_url:str): async with aiohttp.ClientSession() as session: @@ -16,52 +20,85 @@ async def main(room_url:str): room_url, None, "Respond bot", - 5, + 600, ) transport.mic_enabled = True transport.mic_sample_rate = 16000 - transport.camera_enabled = False + transport.camera_enabled = True + transport.camera_width = 1024 + transport.camera_height = 1024 - llm = AzureLLMService() - tts1 = AzureTTSService() - tts2 = ElevenLabsTTSService(session) + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + tts1 = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + tts2 = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl") + dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + + bot1_messages = [ + {"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."}, + ] + bot2_messages = [ + {"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, + ] + + async def get_bot1_statement(): + # Run the LLMs synchronously for the back-and-forth + bot1_msg = await llm.run_llm(bot1_messages) + print(f"bot1_msg: {bot1_msg}") + if bot1_msg: + bot1_messages.append({"role": "assistant", "content": bot1_msg}) + bot2_messages.append({"role": "user", "content": bot1_msg}) + + all_audio = bytearray() + async for audio in tts1.run_tts(bot1_msg): + all_audio.extend(audio) + + return all_audio + + async def get_bot2_statement(): + # Run the LLMs synchronously for the back-and-forth + bot2_msg = await llm.run_llm(bot2_messages) + print(f"bot2_msg: {bot2_msg}") + if bot2_msg: + bot2_messages.append({"role": "assistant", "content": bot2_msg}) + bot1_messages.append({"role": "user", "content": bot2_msg}) + + all_audio = bytearray() + async for audio in tts2.run_tts(bot2_msg): + all_audio.extend(audio) + + return all_audio async def argue(): - bot1_messages = [ - {"role": "system", "content": "You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. Your responses should only be a few sentences long."}, - ] - bot2_messages = [ - {"role": "system", "content": "You strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences."}, - ] - - for i in range(1, 5): + for i in range(100): print(f"In iteration {i}") - # Run the LLMs synchronously for the back-and-forth - bot1_msg = await llm.run_llm(bot1_messages) - print(f"bot1_msg: {bot1_msg}") - if bot1_msg: - bot1_messages.append({"role": "assistant", "content": bot1_msg}) - bot2_messages.append({"role": "user", "content": bot1_msg}) - await tts1.say(bot1_msg, transport.send_queue) + bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed" - bot2_msg = await llm.run_llm(bot2_messages) - print(f"bot2_msg: {bot2_msg}") - if bot2_msg: - bot2_messages.append({"role": "assistant", "content": bot2_msg}) - bot1_messages.append({"role": "user", "content": bot2_msg}) + (audio1, image_data1) = await asyncio.gather( + get_bot1_statement(), dalle.run_image_gen(bot1_description) + ) + await transport.send_queue.put( + [ + ImageQueueFrame(None, image_data1[1]), + AudioQueueFrame(audio1), + ] + ) - await tts2.say(bot2_msg, transport.send_queue) + bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed" + + (audio2, image_data2) = await asyncio.gather( + get_bot2_statement(), dalle.run_image_gen(bot2_description) + ) + await transport.send_queue.put( + [ + ImageQueueFrame(None, image_data2[1]), + AudioQueueFrame(audio2), + ] + ) await asyncio.gather(transport.run(), argue()) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - - asyncio.run(main(args.url)) + (url, token) = configure() + asyncio.run(main(url)) \ No newline at end of file diff --git a/src/samples/foundational/08a-characters-arguing.py b/src/samples/foundational/08a-characters-arguing.py deleted file mode 100644 index 6f247f9f9..000000000 --- a/src/samples/foundational/08a-characters-arguing.py +++ /dev/null @@ -1,108 +0,0 @@ -import aiohttp -import argparse -import asyncio - -from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService -from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -from dailyai.services.fal_ai_services import FalImageGenService -from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame - -async def main(room_url:str): - async with aiohttp.ClientSession() as session: - global transport - global llm - global tts - - transport = DailyTransportService( - room_url, - None, - "Respond bot", - 600, - ) - transport.mic_enabled = True - transport.mic_sample_rate = 16000 - transport.camera_enabled = True - transport.camera_width = 1024 - transport.camera_height = 1024 - - llm = AzureLLMService() - tts1 = AzureTTSService() - tts2 = ElevenLabsTTSService(session) - dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session) - - bot1_messages = [ - {"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."}, - ] - bot2_messages = [ - {"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, - ] - - async def get_bot1_statement(): - # Run the LLMs synchronously for the back-and-forth - bot1_msg = await llm.run_llm(bot1_messages) - print(f"bot1_msg: {bot1_msg}") - if bot1_msg: - bot1_messages.append({"role": "assistant", "content": bot1_msg}) - bot2_messages.append({"role": "user", "content": bot1_msg}) - - all_audio = bytearray() - async for audio in tts1.run_tts(bot1_msg): - all_audio.extend(audio) - - return all_audio - - async def get_bot2_statement(): - # Run the LLMs synchronously for the back-and-forth - bot2_msg = await llm.run_llm(bot2_messages) - print(f"bot2_msg: {bot2_msg}") - if bot2_msg: - bot2_messages.append({"role": "assistant", "content": bot2_msg}) - bot1_messages.append({"role": "user", "content": bot2_msg}) - - all_audio = bytearray() - async for audio in tts2.run_tts(bot2_msg): - all_audio.extend(audio) - - return all_audio - - async def argue(): - for i in range(100): - print(f"In iteration {i}") - - bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed" - - (audio1, image_data1) = await asyncio.gather( - get_bot1_statement(), dalle.run_image_gen(bot1_description) - ) - await transport.send_queue.put( - [ - ImageQueueFrame(None, image_data1[1]), - AudioQueueFrame(audio1), - ] - ) - - bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed" - - (audio2, image_data2) = await asyncio.gather( - get_bot2_statement(), dalle.run_image_gen(bot2_description) - ) - await transport.send_queue.put( - [ - ImageQueueFrame(None, image_data2[1]), - AudioQueueFrame(audio2), - ] - ) - - await asyncio.gather(transport.run(), argue()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - - asyncio.run(main(args.url)) diff --git a/src/samples/foundational/08b-debate-generator.py b/src/samples/foundational/08b-debate-generator.py deleted file mode 100644 index 4366f836d..000000000 --- a/src/samples/foundational/08b-debate-generator.py +++ /dev/null @@ -1,119 +0,0 @@ -import aiohttp -import argparse -import asyncio - -from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService -from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -from dailyai.services.fal_ai_services import FalImageGenService -from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame - -async def main(room_url:str): - async with aiohttp.ClientSession() as session: - global transport - global llm - global tts - - transport = DailyTransportService( - room_url, - None, - "Respond bot", - 600, - ) - transport.mic_enabled = True - transport.mic_sample_rate = 16000 - transport.camera_enabled = True - transport.camera_width = 1024 - transport.camera_height = 1024 - - llm = AzureLLMService() - tts1 = AzureTTSService() - tts2 = ElevenLabsTTSService(session) - dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session) - # dalle = OpenAIImageGenService(image_size="1024x1024") - - topic = "Are pokemon edible?" - affirmative = "A woman dressed as a cowboy, outside on a ranch" - negative = "Pikachu in a business suit" - - # topic = "Is a hot dog a sandwich?" - # affirmative = "A woman conservatively dressed as a librarian in a library surrounded by books" - # negative = "A cat dressed in a hot dog costume" - - - - bot1_messages = [ - {"role": "system", "content": f"You are {affirmative}. You're in a debate, and the topic is: '{topic}'. You're arguing the affirmative. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever agree with the user. Your responses should only be a few sentences long."}, - ] - bot2_messages = [ - {"role": "system", "content": f"You are {negative}. You're in a debate, and the topic is: '{topic}'. You're arguing the negative. Debate this with the user, only responding with a few sentences. Don't ever agree with the user."}, - ] - - async def get_bot1_statement(): - # Run the LLMs synchronously for the back-and-forth - bot1_msg = await llm.run_llm(bot1_messages) - print(f"bot1_msg: {bot1_msg}") - if bot1_msg: - bot1_messages.append({"role": "assistant", "content": bot1_msg}) - bot2_messages.append({"role": "user", "content": bot1_msg}) - - all_audio = bytearray() - async for audio in tts1.run_tts(bot1_msg): - all_audio.extend(audio) - - return all_audio - - async def get_bot2_statement(): - # Run the LLMs synchronously for the back-and-forth - bot2_msg = await llm.run_llm(bot2_messages) - print(f"bot2_msg: {bot2_msg}") - if bot2_msg: - bot2_messages.append({"role": "assistant", "content": bot2_msg}) - bot1_messages.append({"role": "user", "content": bot2_msg}) - - all_audio = bytearray() - async for audio in tts2.run_tts(bot2_msg): - all_audio.extend(audio) - - return all_audio - - async def argue(): - for i in range(100): - print(f"In iteration {i}") - - bot1_description = f"{affirmative}, cartoon, highly detailed" - - (audio1, image_data1) = await asyncio.gather( - get_bot1_statement(), dalle.run_image_gen(bot1_description) - ) - await transport.send_queue.put( - [ - ImageQueueFrame(None, image_data1[1]), - AudioQueueFrame(audio1), - ] - ) - - bot2_description = f"{negative}, cartoon, bright colors, funny, highly detailed" - - (audio2, image_data2) = await asyncio.gather( - get_bot2_statement(), dalle.run_image_gen(bot2_description) - ) - await transport.send_queue.put( - [ - ImageQueueFrame(None, image_data2[1]), - AudioQueueFrame(audio2), - ] - ) - - await asyncio.gather(transport.run(), argue()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - - asyncio.run(main(args.url)) diff --git a/src/samples/foundational/10-wake-word.py b/src/samples/foundational/10-wake-word.py index 6125a2a56..17ae12216 100644 --- a/src/samples/foundational/10-wake-word.py +++ b/src/samples/foundational/10-wake-word.py @@ -1,18 +1,15 @@ import aiohttp -import argparse import asyncio import os import random -import requests -import time -import urllib.parse +from typing import AsyncGenerator from PIL import Image from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService -from dailyai.queue_aggregators import LLMContextAggregator +from dailyai.queue_aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator from dailyai.queue_frame import ( QueueFrame, TextQueueFrame, @@ -22,7 +19,8 @@ from dailyai.queue_frame import ( ) from dailyai.services.ai_services import AIService -from typing import AsyncGenerator +from samples.foundational.support.runner import configure + sprites = {} image_files = [ @@ -123,8 +121,8 @@ async def main(room_url: str, token): transport.camera_width = 720 transport.camera_height = 1280 - llm = AzureLLMService() - tts = ElevenLabsTTSService(session) + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl") isa = ImageSyncAggregator() @transport.event_handler("on_first_other_participant_joined") @@ -136,11 +134,11 @@ async def main(room_url: str, token): {"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."}, ] - tma_in = LLMContextAggregator( - messages, "user", transport.my_participant_id + tma_in = LLMUserContextAggregator( + messages, transport.my_participant_id ) - tma_out = LLMContextAggregator( - messages, "assistant", transport.my_participant_id + tma_out = LLMAssistantContextAggregator( + messages, transport.my_participant_id ) tf = TranscriptFilter(transport.my_participant_id) ncf = NameCheckFilter(["Santa Cat", "Santa"]) @@ -169,35 +167,5 @@ async def main(room_url: str, token): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - parser.add_argument( - "-k", - "--apikey", - type=str, - required=True, - help="Daily API Key (needed to create token)", - ) - - args, unknown = parser.parse_known_args() - - # Create a meeting token for the given room with an expiration 24 hours in the future. - room_name: str = urllib.parse.urlparse(args.url).path[1:] - expiration: float = time.time() + 60 * 60 * 24 - - res: requests.Response = requests.post( - f"https://api.daily.co/v1/meeting-tokens", - headers={"Authorization": f"Bearer {args.apikey}"}, - json={ - "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} - }, - ) - - if res.status_code != 200: - raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") - - token: str = res.json()["token"] - - asyncio.run(main(args.url, token)) + (url, token) = configure() + asyncio.run(main(url, token)) diff --git a/src/samples/foundational/11-sound-effects.py b/src/samples/foundational/11-sound-effects.py index ee913fe56..e2d704a4b 100644 --- a/src/samples/foundational/11-sound-effects.py +++ b/src/samples/foundational/11-sound-effects.py @@ -1,19 +1,19 @@ -import argparse +import aiohttp import asyncio import logging import os import wave -import requests -import time -import urllib.parse from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.queue_aggregators import LLMContextAggregator, LLMUserContextAggregator, LLMAssistantContextAggregator from dailyai.services.ai_services import AIService, FrameLogger from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame from typing import AsyncGenerator +from samples.foundational.support.runner import configure + logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever logger = logging.getLogger("dailyai") logger.setLevel(logging.DEBUG) @@ -64,52 +64,56 @@ class InboundSoundEffectWrapper(AIService): async def main(room_url: str, token): - global transport - global llm - global tts + async with aiohttp.ClientSession() as session: - transport = DailyTransportService( - room_url, - token, - "Respond bot", - 5, - ) - transport.mic_enabled = True - transport.mic_sample_rate = 16000 - transport.camera_enabled = False + global transport + global llm + global tts - llm = AzureLLMService() - tts = AzureTTSService() - - @transport.event_handler("on_first_other_participant_joined") - async def on_first_other_participant_joined(transport): - await tts.say("Hi, I'm listening!", transport.send_queue) - await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) - async def handle_transcriptions(): - messages = [ - {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, - ] - - tma_in = LLMUserContextAggregator( - messages, transport.my_participant_id + transport = DailyTransportService( + room_url, + token, + "Respond bot", + 5, ) - tma_out = LLMAssistantContextAggregator( - messages, transport.my_participant_id - ) - out_sound = OutboundSoundEffectWrapper() - in_sound = InboundSoundEffectWrapper() - fl = FrameLogger("LLM Out") - fl2 = FrameLogger("Transcription In") - await out_sound.run_to_queue( - transport.send_queue, - tts.run( - fl.run( - tma_out.run( - llm.run( - fl2.run( - in_sound.run( - tma_in.run( - transport.get_receive_frames() + transport.mic_enabled = True + transport.mic_sample_rate = 16000 + transport.camera_enabled = False + + llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV") + + + @transport.event_handler("on_first_other_participant_joined") + async def on_first_other_participant_joined(transport): + await tts.say("Hi, I'm listening!", transport.send_queue) + await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) + async def handle_transcriptions(): + messages = [ + {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, + ] + + tma_in = LLMUserContextAggregator( + messages, transport.my_participant_id + ) + tma_out = LLMAssistantContextAggregator( + messages, transport.my_participant_id + ) + out_sound = OutboundSoundEffectWrapper() + in_sound = InboundSoundEffectWrapper() + fl = FrameLogger("LLM Out") + fl2 = FrameLogger("Transcription In") + await out_sound.run_to_queue( + transport.send_queue, + tts.run( + fl.run( + tma_out.run( + llm.run( + fl2.run( + in_sound.run( + tma_in.run( + transport.get_receive_frames() + ) ) ) ) @@ -117,43 +121,12 @@ async def main(room_url: str, token): ) ) ) - ) - + - transport.transcription_settings["extra"]["punctuate"] = True - await asyncio.gather(transport.run(), handle_transcriptions()) + transport.transcription_settings["extra"]["punctuate"] = True + await asyncio.gather(transport.run(), handle_transcriptions()) if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - parser.add_argument( - "-k", - "--apikey", - type=str, - required=True, - help="Daily API Key (needed to create token)", - ) - - args, unknown = parser.parse_known_args() - - # Create a meeting token for the given room with an expiration 1 hour in the future. - room_name: str = urllib.parse.urlparse(args.url).path[1:] - expiration: float = time.time() + 60 * 60 - - res: requests.Response = requests.post( - f"https://api.daily.co/v1/meeting-tokens", - headers={"Authorization": f"Bearer {args.apikey}"}, - json={ - "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} - }, - ) - - if res.status_code != 200: - raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") - - token: str = res.json()["token"] - - asyncio.run(main(args.url, token)) + (url, token) = configure() + asyncio.run(main(url, token)) \ No newline at end of file diff --git a/src/samples/foundational/11a-dial-out.py b/src/samples/foundational/11a-dial-out.py deleted file mode 100644 index 95beb586e..000000000 --- a/src/samples/foundational/11a-dial-out.py +++ /dev/null @@ -1,165 +0,0 @@ -import argparse -import asyncio -import os -import wave -import requests -import time -import urllib.parse - -from dailyai.services.daily_transport_service import DailyTransportService -from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService -from dailyai.queue_aggregators import LLMContextAggregator -from dailyai.services.ai_services import AIService, FrameLogger -from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame -from typing import AsyncGenerator - -sounds = {} -sound_files = [ - 'ding1.wav', - 'ding2.wav' -] - -script_dir = os.path.dirname(__file__) - -for file in sound_files: - # Build the full path to the image file - full_path = os.path.join(script_dir, "assets", file) - # Get the filename without the extension to use as the dictionary key - filename = os.path.splitext(os.path.basename(full_path))[0] - # Open the image and convert it to bytes - with wave.open(full_path) as audio_file: - sounds[file] = audio_file.readframes(-1) - - - - -class OutboundSoundEffectWrapper(AIService): - def __init__(self): - pass - - async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: - if isinstance(frame, LLMResponseEndQueueFrame): - yield AudioQueueFrame(sounds["ding1.wav"]) - # In case anything else up the stack needs it - yield frame - else: - yield frame - -class InboundSoundEffectWrapper(AIService): - def __init__(self): - pass - - async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: - if isinstance(frame, LLMMessagesQueueFrame): - yield AudioQueueFrame(sounds["ding2.wav"]) - # In case anything else up the stack needs it - yield frame - else: - yield frame - - -async def main(room_url: str, token, phone): - global transport - global llm - global tts - - transport = DailyTransportService( - room_url, - token, - "Respond bot", - 300, - ) - transport.mic_enabled = True - transport.mic_sample_rate = 16000 - transport.camera_enabled = False - - llm = AzureLLMService() - tts = AzureTTSService() - - @transport.event_handler("on_first_other_participant_joined") - async def on_first_other_participant_joined(transport): - await tts.say("Hi, I'm listening!", transport.send_queue) - await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) - async def handle_transcriptions(): - messages = [ - {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, - ] - - tma_in = LLMContextAggregator( - messages, "user", transport.my_participant_id - ) - tma_out = LLMContextAggregator( - messages, "assistant", transport.my_participant_id - ) - out_sound = OutboundSoundEffectWrapper() - in_sound = InboundSoundEffectWrapper() - fl = FrameLogger("LLM Out") - fl2 = FrameLogger("Transcription In") - await out_sound.run_to_queue( - transport.send_queue, - tts.run( - tma_out.run( - llm.run( - fl2.run( - in_sound.run( - tma_in.run( - transport.get_receive_frames() - ) - ) - ) - ) - ) - ) - ) - - @transport.event_handler("on_participant_joined") - async def pax_joined(transport, pax): - print(f"PARTICIPANT JOINED: {pax}") - - @transport.event_handler("on_call_state_updated") - async def on_call_state_updated(transport, state): - if (state == "joined"): - if (phone): - transport.start_recording() - transport.dialout(phone) - - - transport.transcription_settings["extra"]["punctuate"] = True - - await asyncio.gather(transport.run(), handle_transcriptions()) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - parser.add_argument( - "-k", - "--apikey", - type=str, - required=True, - help="Daily API Key (needed to create token)", - ) - - parser.add_argument("-p", "--phone", type=str, required=False, help="A phone number to call when the bot joins the room") - - args, unknown = parser.parse_known_args() - - # Create a meeting token for the given room with an expiration 1 hour in the future. - room_name: str = urllib.parse.urlparse(args.url).path[1:] - expiration: float = time.time() + 60 * 60 - - res: requests.Response = requests.post( - f"https://api.staging.daily.co/v1/meeting-tokens", - headers={"Authorization": f"Bearer {args.apikey}"}, - json={ - "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} - }, - ) - - if res.status_code != 200: - raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") - - token: str = res.json()["token"] - asyncio.run(main(args.url, token, args.phone)) diff --git a/src/samples/foundational/13-whisper-transcription.py b/src/samples/foundational/13-whisper-transcription.py index 268f400c7..ff5dc2f28 100644 --- a/src/samples/foundational/13-whisper-transcription.py +++ b/src/samples/foundational/13-whisper-transcription.py @@ -1,9 +1,9 @@ -import argparse import asyncio from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.whisper_ai_services import WhisperSTTService +from samples.foundational.support.runner import configure async def main(room_url: str): global transport @@ -35,10 +35,5 @@ async def main(room_url: str): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") - parser.add_argument( - "-u", "--url", type=str, required=True, help="URL of the Daily room to join" - ) - - args, unknown = parser.parse_known_args() - asyncio.run(main(args.url)) + (url, token) = configure() + asyncio.run(main(url)) \ No newline at end of file diff --git a/src/samples/foundational/support/runner.py b/src/samples/foundational/support/runner.py new file mode 100644 index 000000000..94263c0a6 --- /dev/null +++ b/src/samples/foundational/support/runner.py @@ -0,0 +1,52 @@ +import argparse +import os +import time +import urllib +import requests + +from dotenv import load_dotenv +load_dotenv() + +def configure(): + parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") + parser.add_argument( + "-u", "--url", type=str, required=False, help="URL of the Daily room to join" + ) + parser.add_argument( + "-k", + "--apikey", + type=str, + required=False, + help="Daily API Key (needed to create an owner token for the room)", + ) + + args, unknown = parser.parse_known_args() + + url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL") + key = args.apikey or os.getenv("DAILY_API_KEY") + + if not url: + raise Exception("No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.") + + if not key: + raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.") + + + # Create a meeting token for the given room with an expiration 1 hour in the future. + room_name: str = urllib.parse.urlparse(url).path[1:] + expiration: float = time.time() + 60 * 60 + + res: requests.Response = requests.post( + f"https://api.daily.co/v1/meeting-tokens", + headers={"Authorization": f"Bearer {key}"}, + json={ + "properties": {"room_name": room_name, "is_owner": True, "exp": expiration} + }, + ) + + if res.status_code != 200: + raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}") + + token: str = res.json()["token"] + + return (url, token) \ No newline at end of file diff --git a/src/samples/internal/11a-dial-out.py b/src/samples/internal/11a-dial-out.py new file mode 100644 index 000000000..cd6b39a4d --- /dev/null +++ b/src/samples/internal/11a-dial-out.py @@ -0,0 +1,135 @@ +import aiohttp +import asyncio +import os +import wave + +from dailyai.services.daily_transport_service import DailyTransportService +from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.queue_aggregators import LLMContextAggregator +from dailyai.services.ai_services import AIService, FrameLogger +from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame +from typing import AsyncGenerator + +from samples.foundational.support.runner import configure + +sounds = {} +sound_files = [ + 'ding1.wav', + 'ding2.wav' +] + +script_dir = os.path.dirname(__file__) + +for file in sound_files: + # Build the full path to the image file + full_path = os.path.join(script_dir, "assets", file) + # Get the filename without the extension to use as the dictionary key + filename = os.path.splitext(os.path.basename(full_path))[0] + # Open the image and convert it to bytes + with wave.open(full_path) as audio_file: + sounds[file] = audio_file.readframes(-1) + + + + +class OutboundSoundEffectWrapper(AIService): + def __init__(self): + pass + + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: + if isinstance(frame, LLMResponseEndQueueFrame): + yield AudioQueueFrame(sounds["ding1.wav"]) + # In case anything else up the stack needs it + yield frame + else: + yield frame + +class InboundSoundEffectWrapper(AIService): + def __init__(self): + pass + + async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: + if isinstance(frame, LLMMessagesQueueFrame): + yield AudioQueueFrame(sounds["ding2.wav"]) + # In case anything else up the stack needs it + yield frame + else: + yield frame + + +async def main(room_url: str, token, phone): + async with aiohttp.ClientSession() as session: + + global transport + global llm + global tts + + transport = DailyTransportService( + room_url, + token, + "Respond bot", + 300, + ) + transport.mic_enabled = True + transport.mic_sample_rate = 16000 + transport.camera_enabled = False + + llm = AzureLLMService() + tts = AzureTTSService() + + @transport.event_handler("on_first_other_participant_joined") + async def on_first_other_participant_joined(transport): + await tts.say("Hi, I'm listening!", transport.send_queue) + await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) + async def handle_transcriptions(): + messages = [ + {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, + ] + + tma_in = LLMContextAggregator( + messages, "user", transport.my_participant_id + ) + tma_out = LLMContextAggregator( + messages, "assistant", transport.my_participant_id + ) + out_sound = OutboundSoundEffectWrapper() + in_sound = InboundSoundEffectWrapper() + fl = FrameLogger("LLM Out") + fl2 = FrameLogger("Transcription In") + await out_sound.run_to_queue( + transport.send_queue, + tts.run( + tma_out.run( + llm.run( + fl2.run( + in_sound.run( + tma_in.run( + transport.get_receive_frames() + ) + ) + ) + ) + ) + ) + ) + + @transport.event_handler("on_participant_joined") + async def pax_joined(transport, pax): + print(f"PARTICIPANT JOINED: {pax}") + + @transport.event_handler("on_call_state_updated") + async def on_call_state_updated(transport, state): + if (state == "joined"): + if (phone): + transport.start_recording() + transport.dialout(phone) + + + transport.transcription_settings["extra"]["punctuate"] = True + + await asyncio.gather(transport.run(), handle_transcriptions()) + + +if __name__ == "__main__": + (url, token) = configure() + asyncio.run(main(url, token)) \ No newline at end of file diff --git a/src/samples/server/daily-bot-manager.py b/src/samples/server/daily-bot-manager.py index 9d001cb87..acba09140 100644 --- a/src/samples/server/daily-bot-manager.py +++ b/src/samples/server/daily-bot-manager.py @@ -21,7 +21,7 @@ def start_bot(bot_path, args=None): daily_api_key = os.getenv("DAILY_API_KEY") api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1" - timeout = int(os.getenv("ROOM_TIMEOUT") or os.getenv("BOT_MAX_DURATION") or 300) + timeout = int(os.getenv("DAILY_ROOM_TIMEOUT") or os.getenv("DAILY_BOT_MAX_DURATION") or 300) exp = time.time() + timeout res = requests.post( f"{api_path}/rooms", @@ -82,7 +82,7 @@ def start_bot(bot_path, args=None): # Additional client config config = {} if os.getenv("CLIENT_VAD_TIMEOUT_SEC"): - config['vad_timeout_sec'] = float(os.getenv("CLIENT_VAD_TIMEOUT_SEC")) + config['vad_timeout_sec'] = float(os.getenv("DAILY_CLIENT_VAD_TIMEOUT_SEC")) else: config['vad_timeout_sec'] = 1.5