WIP: environment cleanup (#19)

* removed env var usage from SDK services

* started consolidating configure.py

* 1–3 work

* cleaned up the rest

* more cleanup

* cleanup and 05 tinkering

* made fal keys optional
This commit is contained in:
chadbailey59
2024-02-06 15:07:16 -06:00
committed by GitHub
parent 9d5ad5675c
commit 70d07b6ea2
24 changed files with 475 additions and 804 deletions

View File

@@ -8,6 +8,7 @@ version = "0.0.1"
description = "Orchestrator for AI bots with Daily" description = "Orchestrator for AI bots with Daily"
dependencies = [ dependencies = [
"daily-python", "daily-python",
"python-dotenv",
"Pillow", "Pillow",
"typing-extensions", "typing-extensions",
"openai", "openai",

View File

@@ -17,13 +17,10 @@ from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, Resu
class AzureTTSService(TTSService): class AzureTTSService(TTSService):
def __init__(self, speech_key=None, speech_region=None): def __init__(self, *, api_key, region):
super().__init__() super().__init__()
speech_key = speech_key or os.getenv("AZURE_SPEECH_SERVICE_KEY") self.speech_config = SpeechConfig(subscription=api_key, region=region)
speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION")
self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region)
self.speech_synthesizer = SpeechSynthesizer( self.speech_synthesizer = SpeechSynthesizer(
speech_config=self.speech_config, audio_config=None) speech_config=self.speech_config, audio_config=None)
@@ -51,25 +48,13 @@ class AzureTTSService(TTSService):
class AzureLLMService(LLMService): class AzureLLMService(LLMService):
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None): def __init__(self, *, api_key, endpoint, api_version="2023-12-01-preview", model):
super().__init__() super().__init__()
api_key = api_key or os.getenv("AZURE_CHATGPT_KEY") self._model: str = model
azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT") self._client = AsyncAzureOpenAI(
if not azure_endpoint:
raise Exception(
"No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor")
model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID")
if not model:
raise Exception(
"No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor")
self.model: str = model
api_version = api_version or "2023-12-01-preview"
self.client = AsyncAzureOpenAI(
api_key=api_key, api_key=api_key,
azure_endpoint=azure_endpoint, azure_endpoint=endpoint,
api_version=api_version, api_version=api_version,
) )
@@ -77,7 +62,7 @@ class AzureLLMService(LLMService):
messages_for_log = json.dumps(messages) messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}") self.logger.debug(f"Generating chat via azure: {messages_for_log}")
chunks = await self.client.chat.completions.create(model=self.model, stream=True, messages=messages) chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages)
async for chunk in chunks: async for chunk in chunks:
if len(chunk.choices) == 0: if len(chunk.choices) == 0:
continue continue
@@ -89,7 +74,7 @@ class AzureLLMService(LLMService):
messages_for_log = json.dumps(messages) messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}") self.logger.debug(f"Generating chat via azure: {messages_for_log}")
response = await self.client.chat.completions.create(model=self.model, stream=False, messages=messages) response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0: if response and len(response.choices) > 0:
return response.choices[0].message.content return response.choices[0].message.content
else: else:
@@ -100,17 +85,19 @@ class AzureImageGenServiceREST(ImageGenService):
def __init__( def __init__(
self, self,
*,
api_version="2023-06-01-preview",
image_size: str, image_size: str,
aiohttp_session: aiohttp.ClientSession, aiohttp_session: aiohttp.ClientSession,
api_key=None, api_key,
azure_endpoint=None, endpoint,
api_version=None, model):
model=None):
super().__init__(image_size=image_size) super().__init__(image_size=image_size)
self._api_key = api_key or os.getenv("AZURE_DALLE_KEY")
self._azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT") self._api_key = api_key
self._api_version = api_version or "2023-06-01-preview" self._azure_endpoint = endpoint
self._model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID") self._api_version = api_version
self._model = model
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
async def run_image_gen(self, sentence) -> tuple[str, bytes]: async def run_image_gen(self, sentence) -> tuple[str, bytes]:
@@ -125,8 +112,11 @@ class AzureImageGenServiceREST(ImageGenService):
async with self._aiohttp_session.post( async with self._aiohttp_session.post(
url, headers=headers, json=body url, headers=headers, json=body
) as submission: ) as submission:
print(f"submission: {submission}")
# We never get past this line, because this header isn't
# defined on a 429 response, but something is eating our exceptions!
operation_location = submission.headers['operation-location'] operation_location = submission.headers['operation-location']
print(f"submission status: {submission.status}")
status = "" status = ""
attempts_left = 120 attempts_left = 120
json_response = None json_response = None
@@ -145,9 +135,9 @@ class AzureImageGenServiceREST(ImageGenService):
image_url = json_response["result"]["data"][0]["url"] if json_response else None image_url = json_response["result"]["data"][0]["url"] if json_response else None
if not image_url: if not image_url:
raise Exception("Image generation failed") raise Exception("Image generation failed")
# Load the image from the url # Load the image from the url
async with self._aiohttp_session.get(image_url) as response: async with self._aiohttp_session.get(image_url) as response:
image_stream = io.BytesIO(await response.content.read()) image_stream = io.BytesIO(await response.content.read())
image = Image.open(image_stream) image = Image.open(image_stream)
print("i got an image file!")
return (image_url, image.tobytes()) return (image_url, image.tobytes())

View File

@@ -9,11 +9,11 @@ from dailyai.services.ai_services import TTSService
class DeepgramTTSService(TTSService): class DeepgramTTSService(TTSService):
def __init__(self, aiohttp_session, speech_key=None, voice=None): def __init__(self, *, aiohttp_session, api_key, voice="alpha-asteria-en-v2"):
super().__init__() super().__init__()
self._voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2" self._voice = voice
self._speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY") self._api_key = api_key
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
def get_mic_sample_rate(self): def get_mic_sample_rate(self):
@@ -23,7 +23,7 @@ class DeepgramTTSService(TTSService):
self.logger.info(f"Running deepgram tts for {sentence}") self.logger.info(f"Running deepgram tts for {sentence}")
base_url = "https://api.beta.deepgram.com/v1/speak" base_url = "https://api.beta.deepgram.com/v1/speak"
request_url = f"{base_url}?model={self._voice}&encoding=linear16&container=none&sample_rate=16000" request_url = f"{base_url}?model={self._voice}&encoding=linear16&container=none&sample_rate=16000"
headers = {"authorization": f"token {self._speech_key}"} headers = {"authorization": f"token {self._api_key}"}
body = {"text": sentence} body = {"text": sentence}
async with self._aiohttp_session.post(request_url, headers=headers, json=body) as r: async with self._aiohttp_session.post(request_url, headers=headers, json=body) as r:
async for data in r.content: async for data in r.content:

View File

@@ -12,14 +12,15 @@ class ElevenLabsTTSService(TTSService):
def __init__( def __init__(
self, self,
*,
aiohttp_session: aiohttp.ClientSession, aiohttp_session: aiohttp.ClientSession,
api_key=None, api_key,
voice_id=None, voice_id,
): ):
super().__init__() super().__init__()
self._api_key = api_key or os.getenv("ELEVENLABS_API_KEY") self._api_key = api_key
self._voice_id = voice_id or os.getenv("ELEVENLABS_VOICE_ID") self._voice_id = voice_id
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]: async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:

View File

@@ -2,18 +2,22 @@ import fal
import aiohttp import aiohttp
import asyncio import asyncio
import io import io
import os
import json import json
from PIL import Image from PIL import Image
from dailyai.services.ai_services import LLMService, TTSService, ImageGenService from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
class FalImageGenService(ImageGenService): class FalImageGenService(ImageGenService):
def __init__(self, image_size, aiohttp_session: aiohttp.ClientSession): def __init__(self, *, image_size, aiohttp_session: aiohttp.ClientSession, key_id=None, key_secret=None):
super().__init__(image_size) super().__init__(image_size)
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session
if key_id:
os.environ["FAL_KEY_ID"] = key_id
if key_secret:
os.environ["FAL_KEY_SECRET"] = key_secret
async def run_image_gen(self, sentence) -> tuple[str, bytes]: async def run_image_gen(self, sentence) -> tuple[str, bytes]:
def get_image_url(sentence, size): def get_image_url(sentence, size):

View File

@@ -13,26 +13,24 @@ from dailyai.services.ai_services import AIService, TTSService, LLMService, Imag
class OpenAILLMService(LLMService): class OpenAILLMService(LLMService):
def __init__(self, api_key=None, model=None): def __init__(self, *, api_key, model="gpt-4"):
super().__init__() super().__init__()
api_key = api_key or os.getenv("OPEN_AI_KEY") self._model = model
self.model = model or os.getenv("OPEN_AI_LLM_MODEL") or "gpt-4" self._client = AsyncOpenAI(api_key=api_key)
self.client = AsyncOpenAI(api_key=api_key)
async def get_response(self, messages, stream): async def get_response(self, messages, stream):
return await self.client.chat.completions.create( return await self._client.chat.completions.create(
stream=stream, stream=stream,
messages=messages, messages=messages,
model=self.model model=self._model
) )
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]: async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
messages_for_log = json.dumps(messages) messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}") self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response = await self.get_response(messages, stream=True) chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages)
async for chunk in chunks:
for chunk in response:
if len(chunk.choices) == 0: if len(chunk.choices) == 0:
continue continue
@@ -43,7 +41,7 @@ class OpenAILLMService(LLMService):
messages_for_log = json.dumps(messages) messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}") self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response = await self.get_response(messages, stream=False) response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0: if response and len(response.choices) > 0:
return response.choices[0].message.content return response.choices[0].message.content
else: else:
@@ -54,14 +52,15 @@ class OpenAIImageGenService(ImageGenService):
def __init__( def __init__(
self, self,
*,
image_size: str, image_size: str,
aiohttp_session: aiohttp.ClientSession, aiohttp_session: aiohttp.ClientSession,
api_key=None, api_key,
model=None, model="dall-e-3",
): ):
super().__init__(image_size=image_size) super().__init__(image_size=image_size)
api_key = api_key or os.getenv("OPEN_AI_KEY") self._model = model
self._model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3" print(f"api key: {api_key}")
self._client = AsyncOpenAI(api_key=api_key) self._client = AsyncOpenAI(api_key=api_key)
self._aiohttp_session = aiohttp_session self._aiohttp_session = aiohttp_session

View File

@@ -1,11 +1,11 @@
import argparse
import asyncio import asyncio
import aiohttp import aiohttp
import os
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from samples.foundational.support.runner import configure
async def main(room_url): async def main(room_url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -17,7 +17,7 @@ async def main(room_url):
# #
# the abstract transport service APIs presumably can map pretty closely # the abstract transport service APIs presumably can map pretty closely
# to the daily-python basic API # to the daily-python basic API
meeting_duration_minutes = 1 meeting_duration_minutes = 5
transport = DailyTransportService( transport = DailyTransportService(
room_url, room_url,
None, None,
@@ -25,7 +25,7 @@ async def main(room_url):
meeting_duration_minutes, meeting_duration_minutes,
) )
transport.mic_enabled = True transport.mic_enabled = True
tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV") tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
# Register an event handler so we can play the audio when the participant joins. # Register an event handler so we can play the audio when the participant joins.
@transport.event_handler("on_participant_joined") @transport.event_handler("on_participant_joined")
@@ -45,11 +45,5 @@ async def main(room_url):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,13 +1,16 @@
import argparse import argparse
import asyncio import asyncio
import os
import aiohttp import aiohttp
from dailyai.queue_frame import LLMMessagesQueueFrame from dailyai.queue_frame import LLMMessagesQueueFrame
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from samples.foundational.support.runner import configure
async def main(room_url): async def main(room_url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -20,9 +23,12 @@ async def main(room_url):
) )
transport.mic_enabled = True transport.mic_enabled = True
tts = ElevenLabsTTSService(session, voice_id="29vD33N1CtxCmqQRPOHJ") tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
llm = AzureLLMService() # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE"))
# llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY"))
messages = [{ messages = [{
"role": "system", "role": "system",
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world." "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world."
@@ -43,10 +49,5 @@ async def main(room_url):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,11 +1,15 @@
import argparse import argparse
import asyncio import asyncio
import aiohttp import aiohttp
import os
from dailyai.queue_frame import TextQueueFrame from dailyai.queue_frame import TextQueueFrame
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService
from dailyai.services.azure_ai_services import AzureImageGenServiceREST
from samples.foundational.support.runner import configure
local_joined = False local_joined = False
participant_joined = False participant_joined = False
@@ -25,7 +29,10 @@ async def main(room_url):
transport.camera_width = 1024 transport.camera_width = 1024
transport.camera_height = 1024 transport.camera_height = 1024
imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session) imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
# imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
# imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
image_task = asyncio.create_task( image_task = asyncio.create_task(
imagegen.run_to_queue( imagegen.run_to_queue(
transport.send_queue, [ transport.send_queue, [
@@ -39,11 +46,5 @@ async def main(room_url):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,5 +1,6 @@
import argparse import argparse
import asyncio import asyncio
import os
import re import re
import aiohttp import aiohttp
@@ -9,6 +10,7 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from samples.foundational.support.runner import configure
async def main(room_url: str): async def main(room_url: str):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -22,9 +24,9 @@ async def main(room_url: str):
transport.mic_sample_rate = 16000 transport.mic_sample_rate = 16000
transport.camera_enabled = False transport.camera_enabled = False
llm = AzureLLMService() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
azure_tts = AzureTTSService() azure_tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
elevenlabs_tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV") elevenlabs_tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
messages = [{"role": "system", "content": "tell the user a joke about llamas"}] messages = [{"role": "system", "content": "tell the user a joke about llamas"}]
@@ -60,11 +62,5 @@ async def main(room_url: str):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,14 +1,16 @@
import argparse import argparse
import asyncio import asyncio
import aiohttp import aiohttp
import os
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService
from samples.foundational.support.runner import configure
async def main(room_url): async def main(room_url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -25,10 +27,13 @@ async def main(room_url):
transport.camera_width = 1024 transport.camera_width = 1024
transport.camera_height = 1024 transport.camera_height = 1024
llm = AzureLLMService() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
dalle = FalImageGenService(aiohttp_session=session, image_size="1024x1024") tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV")
tts = ElevenLabsTTSService(aiohttp_session=session, voice_id="ErXwobaYiN019PkySvjV") # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
# dalle = OpenAIImageGenService(image_size="1024x1024")
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
# dalle = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
# dalle = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
# Get a complete audio chunk from the given text. Splitting this into its own # Get a complete audio chunk from the given text. Splitting this into its own
# coroutine lets us ensure proper ordering of the audio chunks on the send queue. # coroutine lets us ensure proper ordering of the audio chunks on the send queue.
@@ -54,10 +59,11 @@ async def main(room_url):
to_speak = f"{month}: {image_description}" to_speak = f"{month}: {image_description}"
audio_task = asyncio.create_task(get_all_audio(to_speak)) audio_task = asyncio.create_task(get_all_audio(to_speak))
image_task = asyncio.create_task(dalle.run_image_gen(image_description)) image_task = asyncio.create_task(dalle.run_image_gen(image_description))
print(f"about to gather tasks for {month}")
(audio, image_data) = await asyncio.gather( (audio, image_data) = await asyncio.gather(
audio_task, image_task audio_task, image_task
) )
print(f"about to return from get_month_data for {month}")
return { return {
"month": month, "month": month,
"text": image_description, "text": image_description,
@@ -72,22 +78,32 @@ async def main(room_url):
"March", "March",
"April", "April",
"May", "May",
"June", "June"
"July",
"August",
"September",
"October",
"November",
"December",
] ]
"""
"February",
"March",
"April",
"May",
"June",
"July",
"August",
"September",
"October",
"November",
"December",
"""
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
# This will play the months in the order they're completed. The benefit # This will play the months in the order they're completed. The benefit
# is we'll have as little delay as possible before the first month, and # is we'll have as little delay as possible before the first month, and
# likely no delay between months, but the months won't display in order. # likely no delay between months, but the months won't display in order.
for month_data_task in asyncio.as_completed(month_tasks): for month_data_task in asyncio.as_completed(month_tasks):
data = await month_data_task print(f"month_data_task: {month_data_task}")
try:
data = await month_data_task
except Exception:
print("OMG EXCEPTION!!!!")
if data: if data:
await transport.send_queue.put( await transport.send_queue.put(
[ [
@@ -104,11 +120,5 @@ async def main(room_url):
await transport.run() await transport.run()
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,13 +1,10 @@
import argparse
import asyncio import asyncio
import requests import os
import time
import urllib.parse
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator
from samples.foundational.support.runner import configure
async def main(room_url: str, token): async def main(room_url: str, token):
global transport global transport
@@ -24,8 +21,8 @@ async def main(room_url: str, token):
transport.mic_sample_rate = 16000 transport.mic_sample_rate = 16000
transport.camera_enabled = False transport.camera_enabled = False
llm = AzureLLMService() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = AzureTTSService() tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
@@ -54,35 +51,5 @@ async def main(room_url: str, token):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url, token))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))

View File

@@ -16,6 +16,7 @@ from dailyai.services.ai_services import AIService
from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from samples.foundational.support.runner import configure
class ImageSyncAggregator(AIService): class ImageSyncAggregator(AIService):
def __init__(self, speaking_path: str, waiting_path: str): def __init__(self, speaking_path: str, waiting_path: str):
@@ -32,7 +33,7 @@ class ImageSyncAggregator(AIService):
async def main(room_url: str, token): async def main(room_url: str, token):
async with aiohttp.ClientSession() as aiohttp_session: async with aiohttp.ClientSession() as session:
transport = DailyTransportService( transport = DailyTransportService(
room_url, room_url,
token, token,
@@ -45,9 +46,9 @@ async def main(room_url: str, token):
transport.mic_enabled = True transport.mic_enabled = True
transport.mic_sample_rate = 16000 transport.mic_sample_rate = 16000
llm = AzureLLMService() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = AzureTTSService() tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
img = FalImageGenService(image_size="1024x1024", aiohttp_session=aiohttp_session) img = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
async def get_images(): async def get_images():
get_speaking_task = asyncio.create_task( get_speaking_task = asyncio.create_task(
@@ -100,35 +101,5 @@ async def main(room_url: str, token):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url, token))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))

View File

@@ -1,16 +1,14 @@
import argparse
import asyncio import asyncio
import aiohttp import aiohttp
import requests import os
import time
import urllib.parse
from dailyai.conversation_wrappers import InterruptibleConversationWrapper from dailyai.conversation_wrappers import InterruptibleConversationWrapper
from dailyai.queue_frame import StartStreamQueueFrame, TextQueueFrame from dailyai.queue_frame import StartStreamQueueFrame, TextQueueFrame
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from samples.foundational.support.runner import configure
async def main(room_url: str, token): async def main(room_url: str, token):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -25,8 +23,8 @@ async def main(room_url: str, token):
transport.camera_enabled = False transport.camera_enabled = False
transport.start_transcription = True transport.start_transcription = True
llm = AzureLLMService() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV") tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
async def run_response(user_speech, tma_in, tma_out): async def run_response(user_speech, tma_in, tma_out):
await tts.run_to_queue( await tts.run_to_queue(
@@ -63,35 +61,5 @@ async def main(room_url: str, token):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url, token))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))

View File

@@ -1,10 +1,14 @@
import aiohttp import aiohttp
import argparse
import asyncio import asyncio
import os
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
from samples.foundational.support.runner import configure
async def main(room_url:str): async def main(room_url:str):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -16,52 +20,85 @@ async def main(room_url:str):
room_url, room_url,
None, None,
"Respond bot", "Respond bot",
5, 600,
) )
transport.mic_enabled = True transport.mic_enabled = True
transport.mic_sample_rate = 16000 transport.mic_sample_rate = 16000
transport.camera_enabled = False transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts1 = AzureTTSService() tts1 = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
tts2 = ElevenLabsTTSService(session) tts2 = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl")
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
bot1_messages = [
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."},
]
async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio)
return all_audio
async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}")
if bot2_msg:
bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio)
return all_audio
async def argue(): async def argue():
bot1_messages = [ for i in range(100):
{"role": "system", "content": "You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": "You strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences."},
]
for i in range(1, 5):
print(f"In iteration {i}") print(f"In iteration {i}")
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
await tts1.say(bot1_msg, transport.send_queue) bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed"
bot2_msg = await llm.run_llm(bot2_messages) (audio1, image_data1) = await asyncio.gather(
print(f"bot2_msg: {bot2_msg}") get_bot1_statement(), dalle.run_image_gen(bot1_description)
if bot2_msg: )
bot2_messages.append({"role": "assistant", "content": bot2_msg}) await transport.send_queue.put(
bot1_messages.append({"role": "user", "content": bot2_msg}) [
ImageQueueFrame(None, image_data1[1]),
AudioQueueFrame(audio1),
]
)
await tts2.say(bot2_msg, transport.send_queue) bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data2[1]),
AudioQueueFrame(audio2),
]
)
await asyncio.gather(transport.run(), argue()) await asyncio.gather(transport.run(), argue())
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,108 +0,0 @@
import aiohttp
import argparse
import asyncio
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
async def main(room_url:str):
async with aiohttp.ClientSession() as session:
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
None,
"Respond bot",
600,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService()
tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService(session)
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session)
bot1_messages = [
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."},
]
async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio)
return all_audio
async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}")
if bot2_msg:
bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio)
return all_audio
async def argue():
for i in range(100):
print(f"In iteration {i}")
bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed"
(audio1, image_data1) = await asyncio.gather(
get_bot1_statement(), dalle.run_image_gen(bot1_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data1[1]),
AudioQueueFrame(audio1),
]
)
bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data2[1]),
AudioQueueFrame(audio2),
]
)
await asyncio.gather(transport.run(), argue())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,119 +0,0 @@
import aiohttp
import argparse
import asyncio
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
async def main(room_url:str):
async with aiohttp.ClientSession() as session:
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
None,
"Respond bot",
600,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService()
tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService(session)
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session)
# dalle = OpenAIImageGenService(image_size="1024x1024")
topic = "Are pokemon edible?"
affirmative = "A woman dressed as a cowboy, outside on a ranch"
negative = "Pikachu in a business suit"
# topic = "Is a hot dog a sandwich?"
# affirmative = "A woman conservatively dressed as a librarian in a library surrounded by books"
# negative = "A cat dressed in a hot dog costume"
bot1_messages = [
{"role": "system", "content": f"You are {affirmative}. You're in a debate, and the topic is: '{topic}'. You're arguing the affirmative. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever agree with the user. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": f"You are {negative}. You're in a debate, and the topic is: '{topic}'. You're arguing the negative. Debate this with the user, only responding with a few sentences. Don't ever agree with the user."},
]
async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio)
return all_audio
async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}")
if bot2_msg:
bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio)
return all_audio
async def argue():
for i in range(100):
print(f"In iteration {i}")
bot1_description = f"{affirmative}, cartoon, highly detailed"
(audio1, image_data1) = await asyncio.gather(
get_bot1_statement(), dalle.run_image_gen(bot1_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data1[1]),
AudioQueueFrame(audio1),
]
)
bot2_description = f"{negative}, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data2[1]),
AudioQueueFrame(audio2),
]
)
await asyncio.gather(transport.run(), argue())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -1,18 +1,15 @@
import aiohttp import aiohttp
import argparse
import asyncio import asyncio
import os import os
import random import random
import requests from typing import AsyncGenerator
import time
import urllib.parse
from PIL import Image from PIL import Image
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.queue_aggregators import LLMContextAggregator from dailyai.queue_aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator
from dailyai.queue_frame import ( from dailyai.queue_frame import (
QueueFrame, QueueFrame,
TextQueueFrame, TextQueueFrame,
@@ -22,7 +19,8 @@ from dailyai.queue_frame import (
) )
from dailyai.services.ai_services import AIService from dailyai.services.ai_services import AIService
from typing import AsyncGenerator from samples.foundational.support.runner import configure
sprites = {} sprites = {}
image_files = [ image_files = [
@@ -123,8 +121,8 @@ async def main(room_url: str, token):
transport.camera_width = 720 transport.camera_width = 720
transport.camera_height = 1280 transport.camera_height = 1280
llm = AzureLLMService() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = ElevenLabsTTSService(session) tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl")
isa = ImageSyncAggregator() isa = ImageSyncAggregator()
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
@@ -136,11 +134,11 @@ async def main(room_url: str, token):
{"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."}, {"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."},
] ]
tma_in = LLMContextAggregator( tma_in = LLMUserContextAggregator(
messages, "user", transport.my_participant_id messages, transport.my_participant_id
) )
tma_out = LLMContextAggregator( tma_out = LLMAssistantContextAggregator(
messages, "assistant", transport.my_participant_id messages, transport.my_participant_id
) )
tf = TranscriptFilter(transport.my_participant_id) tf = TranscriptFilter(transport.my_participant_id)
ncf = NameCheckFilter(["Santa Cat", "Santa"]) ncf = NameCheckFilter(["Santa Cat", "Santa"])
@@ -169,35 +167,5 @@ async def main(room_url: str, token):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url, token))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 24 hours in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60 * 24
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))

View File

@@ -1,19 +1,19 @@
import argparse import aiohttp
import asyncio import asyncio
import logging import logging
import os import os
import wave import wave
import requests
import time
import urllib.parse
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.queue_aggregators import LLMContextAggregator, LLMUserContextAggregator, LLMAssistantContextAggregator from dailyai.queue_aggregators import LLMContextAggregator, LLMUserContextAggregator, LLMAssistantContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator from typing import AsyncGenerator
from samples.foundational.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever
logger = logging.getLogger("dailyai") logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
@@ -64,52 +64,56 @@ class InboundSoundEffectWrapper(AIService):
async def main(room_url: str, token): async def main(room_url: str, token):
global transport async with aiohttp.ClientSession() as session:
global llm
global tts
transport = DailyTransportService( global transport
room_url, global llm
token, global tts
"Respond bot",
5,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService() transport = DailyTransportService(
tts = AzureTTSService() room_url,
token,
@transport.event_handler("on_first_other_participant_joined") "Respond bot",
async def on_first_other_participant_joined(transport): 5,
await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMUserContextAggregator(
messages, transport.my_participant_id
) )
tma_out = LLMAssistantContextAggregator( transport.mic_enabled = True
messages, transport.my_participant_id transport.mic_sample_rate = 16000
) transport.camera_enabled = False
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper() llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
fl = FrameLogger("LLM Out") tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue, @transport.event_handler("on_first_other_participant_joined")
tts.run( async def on_first_other_participant_joined(transport):
fl.run( await tts.say("Hi, I'm listening!", transport.send_queue)
tma_out.run( await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
llm.run( async def handle_transcriptions():
fl2.run( messages = [
in_sound.run( {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
tma_in.run( ]
transport.get_receive_frames()
tma_in = LLMUserContextAggregator(
messages, transport.my_participant_id
)
tma_out = LLMAssistantContextAggregator(
messages, transport.my_participant_id
)
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper()
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue,
tts.run(
fl.run(
tma_out.run(
llm.run(
fl2.run(
in_sound.run(
tma_in.run(
transport.get_receive_frames()
)
) )
) )
) )
@@ -117,43 +121,12 @@ async def main(room_url: str, token):
) )
) )
) )
)
transport.transcription_settings["extra"]["punctuate"] = True transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions()) await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url, token))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))

View File

@@ -1,165 +0,0 @@
import argparse
import asyncio
import os
import wave
import requests
import time
import urllib.parse
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_aggregators import LLMContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator
sounds = {}
sound_files = [
'ding1.wav',
'ding2.wav'
]
script_dir = os.path.dirname(__file__)
for file in sound_files:
# Build the full path to the image file
full_path = os.path.join(script_dir, "assets", file)
# Get the filename without the extension to use as the dictionary key
filename = os.path.splitext(os.path.basename(full_path))[0]
# Open the image and convert it to bytes
with wave.open(full_path) as audio_file:
sounds[file] = audio_file.readframes(-1)
class OutboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMResponseEndQueueFrame):
yield AudioQueueFrame(sounds["ding1.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
class InboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
yield AudioQueueFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
async def main(room_url: str, token, phone):
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
token,
"Respond bot",
300,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService()
tts = AzureTTSService()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMContextAggregator(
messages, "user", transport.my_participant_id
)
tma_out = LLMContextAggregator(
messages, "assistant", transport.my_participant_id
)
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper()
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue,
tts.run(
tma_out.run(
llm.run(
fl2.run(
in_sound.run(
tma_in.run(
transport.get_receive_frames()
)
)
)
)
)
)
)
@transport.event_handler("on_participant_joined")
async def pax_joined(transport, pax):
print(f"PARTICIPANT JOINED: {pax}")
@transport.event_handler("on_call_state_updated")
async def on_call_state_updated(transport, state):
if (state == "joined"):
if (phone):
transport.start_recording()
transport.dialout(phone)
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
parser.add_argument("-p", "--phone", type=str, required=False, help="A phone number to call when the bot joins the room")
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.staging.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token, args.phone))

View File

@@ -1,9 +1,9 @@
import argparse
import asyncio import asyncio
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.whisper_ai_services import WhisperSTTService from dailyai.services.whisper_ai_services import WhisperSTTService
from samples.foundational.support.runner import configure
async def main(room_url: str): async def main(room_url: str):
global transport global transport
@@ -35,10 +35,5 @@ async def main(room_url: str):
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") (url, token) = configure()
parser.add_argument( asyncio.run(main(url))
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

View File

@@ -0,0 +1,52 @@
import argparse
import os
import time
import urllib
import requests
from dotenv import load_dotenv
load_dotenv()
def configure():
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=False, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=False,
help="Daily API Key (needed to create an owner token for the room)",
)
args, unknown = parser.parse_known_args()
url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
key = args.apikey or os.getenv("DAILY_API_KEY")
if not url:
raise Exception("No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.")
if not key:
raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {key}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
return (url, token)

View File

@@ -0,0 +1,135 @@
import aiohttp
import asyncio
import os
import wave
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_aggregators import LLMContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator
from samples.foundational.support.runner import configure
sounds = {}
sound_files = [
'ding1.wav',
'ding2.wav'
]
script_dir = os.path.dirname(__file__)
for file in sound_files:
# Build the full path to the image file
full_path = os.path.join(script_dir, "assets", file)
# Get the filename without the extension to use as the dictionary key
filename = os.path.splitext(os.path.basename(full_path))[0]
# Open the image and convert it to bytes
with wave.open(full_path) as audio_file:
sounds[file] = audio_file.readframes(-1)
class OutboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMResponseEndQueueFrame):
yield AudioQueueFrame(sounds["ding1.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
class InboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
yield AudioQueueFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
async def main(room_url: str, token, phone):
async with aiohttp.ClientSession() as session:
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
token,
"Respond bot",
300,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService()
tts = AzureTTSService()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMContextAggregator(
messages, "user", transport.my_participant_id
)
tma_out = LLMContextAggregator(
messages, "assistant", transport.my_participant_id
)
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper()
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue,
tts.run(
tma_out.run(
llm.run(
fl2.run(
in_sound.run(
tma_in.run(
transport.get_receive_frames()
)
)
)
)
)
)
)
@transport.event_handler("on_participant_joined")
async def pax_joined(transport, pax):
print(f"PARTICIPANT JOINED: {pax}")
@transport.event_handler("on_call_state_updated")
async def on_call_state_updated(transport, state):
if (state == "joined"):
if (phone):
transport.start_recording()
transport.dialout(phone)
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__":
(url, token) = configure()
asyncio.run(main(url, token))

View File

@@ -21,7 +21,7 @@ def start_bot(bot_path, args=None):
daily_api_key = os.getenv("DAILY_API_KEY") daily_api_key = os.getenv("DAILY_API_KEY")
api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1" api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1"
timeout = int(os.getenv("ROOM_TIMEOUT") or os.getenv("BOT_MAX_DURATION") or 300) timeout = int(os.getenv("DAILY_ROOM_TIMEOUT") or os.getenv("DAILY_BOT_MAX_DURATION") or 300)
exp = time.time() + timeout exp = time.time() + timeout
res = requests.post( res = requests.post(
f"{api_path}/rooms", f"{api_path}/rooms",
@@ -82,7 +82,7 @@ def start_bot(bot_path, args=None):
# Additional client config # Additional client config
config = {} config = {}
if os.getenv("CLIENT_VAD_TIMEOUT_SEC"): if os.getenv("CLIENT_VAD_TIMEOUT_SEC"):
config['vad_timeout_sec'] = float(os.getenv("CLIENT_VAD_TIMEOUT_SEC")) config['vad_timeout_sec'] = float(os.getenv("DAILY_CLIENT_VAD_TIMEOUT_SEC"))
else: else:
config['vad_timeout_sec'] = 1.5 config['vad_timeout_sec'] = 1.5