wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP: environment cleanup (#19)

Browse Source 
* removed env var usage from SDK services

* started consolidating configure.py

* 1–3 work

* cleaned up the rest

* more cleanup

* cleanup and 05 tinkering

* made fal keys optional
This commit is contained in:
chadbailey59
2024-02-06 15:07:16 -06:00
committed by GitHub
parent 9d5ad5675c
commit 70d07b6ea2
24 changed files with 475 additions and 804 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
pyproject.toml
View File

@@ -8,6 +8,7 @@ version = "0.0.1"
description = "Orchestrator for AI bots with Daily"
dependencies = [
"daily-python",
"python-dotenv",
"Pillow",
"typing-extensions",
"openai",

56
src/dailyai/services/azure_ai_services.py
View File

@@ -17,13 +17,10 @@ from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, Resu
class AzureTTSService(TTSService):
def __init__(self, speech_key=None, speech_region=None):
def __init__(self, *, api_key, region):
super().__init__()
speech_key = speech_key or os.getenv("AZURE_SPEECH_SERVICE_KEY")
speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION")
self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region)
self.speech_config = SpeechConfig(subscription=api_key, region=region)
self.speech_synthesizer = SpeechSynthesizer(
speech_config=self.speech_config, audio_config=None)
@@ -51,25 +48,13 @@ class AzureTTSService(TTSService):
class AzureLLMService(LLMService):
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None):
def __init__(self, *, api_key, endpoint, api_version="2023-12-01-preview", model):
super().__init__()
api_key = api_key or os.getenv("AZURE_CHATGPT_KEY")
self._model: str = model
azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT")
if not azure_endpoint:
raise Exception(
"No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor")
model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID")
if not model:
raise Exception(
"No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor")
self.model: str = model
api_version = api_version or "2023-12-01-preview"
self.client = AsyncAzureOpenAI(
self._client = AsyncAzureOpenAI(
api_key=api_key,
azure_endpoint=azure_endpoint,
azure_endpoint=endpoint,
api_version=api_version,
)
@@ -77,7 +62,7 @@ class AzureLLMService(LLMService):
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}")
chunks = await self.client.chat.completions.create(model=self.model, stream=True, messages=messages)
chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages)
async for chunk in chunks:
if len(chunk.choices) == 0:
continue
@@ -89,7 +74,7 @@ class AzureLLMService(LLMService):
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via azure: {messages_for_log}")
response = await self.client.chat.completions.create(model=self.model, stream=False, messages=messages)
response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0:
return response.choices[0].message.content
else:
@@ -100,17 +85,19 @@ class AzureImageGenServiceREST(ImageGenService):
def __init__(
self,
*,
api_version="2023-06-01-preview",
image_size: str,
aiohttp_session: aiohttp.ClientSession,
api_key=None,
azure_endpoint=None,
api_version=None,
model=None):
api_key,
endpoint,
model):
super().__init__(image_size=image_size)
self._api_key = api_key or os.getenv("AZURE_DALLE_KEY")
self._azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
self._api_version = api_version or "2023-06-01-preview"
self._model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID")
self._api_key = api_key
self._azure_endpoint = endpoint
self._api_version = api_version
self._model = model
self._aiohttp_session = aiohttp_session
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
@@ -125,8 +112,11 @@ class AzureImageGenServiceREST(ImageGenService):
async with self._aiohttp_session.post(
url, headers=headers, json=body
) as submission:
print(f"submission: {submission}")
# We never get past this line, because this header isn't
# defined on a 429 response, but something is eating our exceptions!
operation_location = submission.headers['operation-location']
print(f"submission status: {submission.status}")
status = ""
attempts_left = 120
json_response = None
@@ -145,9 +135,9 @@ class AzureImageGenServiceREST(ImageGenService):
image_url = json_response["result"]["data"][0]["url"] if json_response else None
if not image_url:
raise Exception("Image generation failed")
# Load the image from the url
async with self._aiohttp_session.get(image_url) as response:
image_stream = io.BytesIO(await response.content.read())
image = Image.open(image_stream)
print("i got an image file!")
return (image_url, image.tobytes())

8
src/dailyai/services/deepgram_ai_services.py
View File

@@ -9,11 +9,11 @@ from dailyai.services.ai_services import TTSService
class DeepgramTTSService(TTSService):
def __init__(self, aiohttp_session, speech_key=None, voice=None):
def __init__(self, *, aiohttp_session, api_key, voice="alpha-asteria-en-v2"):
super().__init__()
self._voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2"
self._speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY")
self._voice = voice
self._api_key = api_key
self._aiohttp_session = aiohttp_session
def get_mic_sample_rate(self):
@@ -23,7 +23,7 @@ class DeepgramTTSService(TTSService):
self.logger.info(f"Running deepgram tts for {sentence}")
base_url = "https://api.beta.deepgram.com/v1/speak"
request_url = f"{base_url}?model={self._voice}&encoding=linear16&container=none&sample_rate=16000"
headers = {"authorization": f"token {self._speech_key}"}
headers = {"authorization": f"token {self._api_key}"}
body = {"text": sentence}
async with self._aiohttp_session.post(request_url, headers=headers, json=body) as r:
async for data in r.content:

9
src/dailyai/services/elevenlabs_ai_service.py
View File

@@ -12,14 +12,15 @@ class ElevenLabsTTSService(TTSService):
def __init__(
self,
*,
aiohttp_session: aiohttp.ClientSession,
api_key=None,
voice_id=None,
api_key,
voice_id,
):
super().__init__()
self._api_key = api_key or os.getenv("ELEVENLABS_API_KEY")
self._voice_id = voice_id or os.getenv("ELEVENLABS_VOICE_ID")
self._api_key = api_key
self._voice_id = voice_id
self._aiohttp_session = aiohttp_session
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:

8
src/dailyai/services/fal_ai_services.py
View File

@@ -2,18 +2,22 @@ import fal
import aiohttp
import asyncio
import io
import os
import json
from PIL import Image
from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
class FalImageGenService(ImageGenService):
def __init__(self, image_size, aiohttp_session: aiohttp.ClientSession):
def __init__(self, *, image_size, aiohttp_session: aiohttp.ClientSession, key_id=None, key_secret=None):
super().__init__(image_size)
self._aiohttp_session = aiohttp_session
if key_id:
os.environ["FAL_KEY_ID"] = key_id
if key_secret:
os.environ["FAL_KEY_SECRET"] = key_secret
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
def get_image_url(sentence, size):

27
src/dailyai/services/open_ai_services.py
View File

@@ -13,26 +13,24 @@ from dailyai.services.ai_services import AIService, TTSService, LLMService, Imag
class OpenAILLMService(LLMService):
def __init__(self, api_key=None, model=None):
def __init__(self, *, api_key, model="gpt-4"):
super().__init__()
api_key = api_key or os.getenv("OPEN_AI_KEY")
self.model = model or os.getenv("OPEN_AI_LLM_MODEL") or "gpt-4"
self.client = AsyncOpenAI(api_key=api_key)
self._model = model
self._client = AsyncOpenAI(api_key=api_key)
async def get_response(self, messages, stream):
return await self.client.chat.completions.create(
return await self._client.chat.completions.create(
stream=stream,
messages=messages,
model=self.model
model=self._model
)
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response = await self.get_response(messages, stream=True)
for chunk in response:
chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages)
async for chunk in chunks:
if len(chunk.choices) == 0:
continue
@@ -43,7 +41,7 @@ class OpenAILLMService(LLMService):
messages_for_log = json.dumps(messages)
self.logger.debug(f"Generating chat via openai: {messages_for_log}")
response = await self.get_response(messages, stream=False)
response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages)
if response and len(response.choices) > 0:
return response.choices[0].message.content
else:
@@ -54,14 +52,15 @@ class OpenAIImageGenService(ImageGenService):
def __init__(
self,
*,
image_size: str,
aiohttp_session: aiohttp.ClientSession,
api_key=None,
model=None,
api_key,
model="dall-e-3",
):
super().__init__(image_size=image_size)
api_key = api_key or os.getenv("OPEN_AI_KEY")
self._model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3"
self._model = model
print(f"api key: {api_key}")
self._client = AsyncOpenAI(api_key=api_key)
self._aiohttp_session = aiohttp_session

18
src/samples/foundational/01-say-one-thing.py
View File

@@ -1,11 +1,11 @@
import argparse
import asyncio
import aiohttp
import os
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from samples.foundational.support.runner import configure
async def main(room_url):
async with aiohttp.ClientSession() as session:
@@ -17,7 +17,7 @@ async def main(room_url):
#
# the abstract transport service APIs presumably can map pretty closely
# to the daily-python basic API
meeting_duration_minutes = 1
meeting_duration_minutes = 5
transport = DailyTransportService(
room_url,
None,
@@ -25,7 +25,7 @@ async def main(room_url):
meeting_duration_minutes,
)
transport.mic_enabled = True
tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV")
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
# Register an event handler so we can play the audio when the participant joins.
@transport.event_handler("on_participant_joined")
@@ -45,11 +45,5 @@ async def main(room_url):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))
(url, token) = configure()
asyncio.run(main(url))

25
src/samples/foundational/02-llm-say-one-thing.py
View File

@@ -1,13 +1,16 @@
import argparse
import asyncio
import os
import aiohttp
from dailyai.queue_frame import LLMMessagesQueueFrame
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from samples.foundational.support.runner import configure
async def main(room_url):
async with aiohttp.ClientSession() as session:
@@ -20,9 +23,12 @@ async def main(room_url):
)
transport.mic_enabled = True
tts = ElevenLabsTTSService(session, voice_id="29vD33N1CtxCmqQRPOHJ")
llm = AzureLLMService()
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
# tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE"))
# llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY"))
messages = [{
"role": "system",
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world."
@@ -43,10 +49,5 @@ async def main(room_url):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))
(url, token) = configure()
asyncio.run(main(url))

21
src/samples/foundational/03-still-frame.py
View File

@@ -1,11 +1,15 @@
import argparse
import asyncio
import aiohttp
import os
from dailyai.queue_frame import TextQueueFrame
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService
from dailyai.services.azure_ai_services import AzureImageGenServiceREST
from samples.foundational.support.runner import configure
local_joined = False
participant_joined = False
@@ -25,7 +29,10 @@ async def main(room_url):
transport.camera_width = 1024
transport.camera_height = 1024
imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session)
imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
# imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
# imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
image_task = asyncio.create_task(
imagegen.run_to_queue(
transport.send_queue, [
@@ -39,11 +46,5 @@ async def main(room_url):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))
(url, token) = configure()
asyncio.run(main(url))

18
src/samples/foundational/04-utterance-and-speech.py
View File

@@ -1,5 +1,6 @@
import argparse
import asyncio
import os
import re
import aiohttp
@@ -9,6 +10,7 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from samples.foundational.support.runner import configure
async def main(room_url: str):
async with aiohttp.ClientSession() as session:
@@ -22,9 +24,9 @@ async def main(room_url: str):
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService()
azure_tts = AzureTTSService()
elevenlabs_tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV")
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
azure_tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
elevenlabs_tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
messages = [{"role": "system", "content": "tell the user a joke about llamas"}]
@@ -60,11 +62,5 @@ async def main(room_url: str):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))
(url, token) = configure()
asyncio.run(main(url))

58
src/samples/foundational/05-sync-speech-and-text.py → src/samples/foundational/05-sync-speech-and-image.py
View File

@@ -1,14 +1,16 @@
import argparse
import asyncio
import aiohttp
import os
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService
from samples.foundational.support.runner import configure
async def main(room_url):
async with aiohttp.ClientSession() as session:
@@ -25,11 +27,14 @@ async def main(room_url):
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService()
dalle = FalImageGenService(aiohttp_session=session, image_size="1024x1024")
tts = ElevenLabsTTSService(aiohttp_session=session, voice_id="ErXwobaYiN019PkySvjV")
# dalle = OpenAIImageGenService(image_size="1024x1024")
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV")
# tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
# dalle = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
# dalle = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
# Get a complete audio chunk from the given text. Splitting this into its own
# coroutine lets us ensure proper ordering of the audio chunks on the send queue.
async def get_all_audio(text):
@@ -54,10 +59,11 @@ async def main(room_url):
to_speak = f"{month}: {image_description}"
audio_task = asyncio.create_task(get_all_audio(to_speak))
image_task = asyncio.create_task(dalle.run_image_gen(image_description))
print(f"about to gather tasks for {month}")
(audio, image_data) = await asyncio.gather(
audio_task, image_task
)
print(f"about to return from get_month_data for {month}")
return {
"month": month,
"text": image_description,
@@ -72,22 +78,32 @@ async def main(room_url):
"March",
"April",
"May",
"June",
"July",
"August",
"September",
"October",
"November",
"December",
"June"
]
"""
"February",
"March",
"April",
"May",
"June",
"July",
"August",
"September",
"October",
"November",
"December",
"""
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
# This will play the months in the order they're completed. The benefit
# is we'll have as little delay as possible before the first month, and
# likely no delay between months, but the months won't display in order.
for month_data_task in asyncio.as_completed(month_tasks):
data = await month_data_task
print(f"month_data_task: {month_data_task}")
try:
data = await month_data_task
except Exception:
print("OMG EXCEPTION!!!!")
if data:
await transport.send_queue.put(
[
@@ -104,11 +120,5 @@ async def main(room_url):
await transport.run()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))
(url, token) = configure()
asyncio.run(main(url))

45
src/samples/foundational/06-listen-and-respond.py
View File

@@ -1,13 +1,10 @@
import argparse
import asyncio
import requests
import time
import urllib.parse
import os
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator
from samples.foundational.support.runner import configure
async def main(room_url: str, token):
global transport
@@ -24,8 +21,8 @@ async def main(room_url: str, token):
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService()
tts = AzureTTSService()
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
@@ -54,35 +51,5 @@ async def main(room_url: str, token):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))
(url, token) = configure()
asyncio.run(main(url, token))

43
src/samples/foundational/06a-image-sync.py
View File

@@ -16,6 +16,7 @@ from dailyai.services.ai_services import AIService
from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator
from dailyai.services.fal_ai_services import FalImageGenService
from samples.foundational.support.runner import configure
class ImageSyncAggregator(AIService):
def __init__(self, speaking_path: str, waiting_path: str):
@@ -32,7 +33,7 @@ class ImageSyncAggregator(AIService):
async def main(room_url: str, token):
async with aiohttp.ClientSession() as aiohttp_session:
async with aiohttp.ClientSession() as session:
transport = DailyTransportService(
room_url,
token,
@@ -45,9 +46,9 @@ async def main(room_url: str, token):
transport.mic_enabled = True
transport.mic_sample_rate = 16000
llm = AzureLLMService()
tts = AzureTTSService()
img = FalImageGenService(image_size="1024x1024", aiohttp_session=aiohttp_session)
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
img = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
async def get_images():
get_speaking_task = asyncio.create_task(
@@ -100,35 +101,5 @@ async def main(room_url: str, token):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))
(url, token) = configure()
asyncio.run(main(url, token))

46
src/samples/foundational/07-interruptible.py
View File

@@ -1,16 +1,14 @@
import argparse
import asyncio
import aiohttp
import requests
import time
import urllib.parse
import os
from dailyai.conversation_wrappers import InterruptibleConversationWrapper
from dailyai.queue_frame import StartStreamQueueFrame, TextQueueFrame
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from samples.foundational.support.runner import configure
async def main(room_url: str, token):
async with aiohttp.ClientSession() as session:
@@ -25,8 +23,8 @@ async def main(room_url: str, token):
transport.camera_enabled = False
transport.start_transcription = True
llm = AzureLLMService()
tts = ElevenLabsTTSService(session, voice_id="ErXwobaYiN019PkySvjV")
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
async def run_response(user_speech, tma_in, tma_out):
await tts.run_to_queue(
@@ -63,35 +61,5 @@ async def main(room_url: str, token):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))
(url, token) = configure()
asyncio.run(main(url, token))

107
src/samples/foundational/08-bots-arguing.py
View File

@@ -1,10 +1,14 @@
import aiohttp
import argparse
import asyncio
import os
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
from samples.foundational.support.runner import configure
async def main(room_url:str):
async with aiohttp.ClientSession() as session:
@@ -16,52 +20,85 @@ async def main(room_url:str):
room_url,
None,
"Respond bot",
5,
600,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService()
tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService(session)
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts1 = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
tts2 = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl")
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET"))
bot1_messages = [
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."},
]
async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio)
return all_audio
async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}")
if bot2_msg:
bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio)
return all_audio
async def argue():
bot1_messages = [
{"role": "system", "content": "You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": "You strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences."},
]
for i in range(1, 5):
for i in range(100):
print(f"In iteration {i}")
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
await tts1.say(bot1_msg, transport.send_queue)
bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed"
bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}")
if bot2_msg:
bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg})
(audio1, image_data1) = await asyncio.gather(
get_bot1_statement(), dalle.run_image_gen(bot1_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data1[1]),
AudioQueueFrame(audio1),
]
)
await tts2.say(bot2_msg, transport.send_queue)
bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data2[1]),
AudioQueueFrame(audio2),
]
)
await asyncio.gather(transport.run(), argue())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))
(url, token) = configure()
asyncio.run(main(url))

108
src/samples/foundational/08a-characters-arguing.py
View File

@@ -1,108 +0,0 @@
import aiohttp
import argparse
import asyncio
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
async def main(room_url:str):
async with aiohttp.ClientSession() as session:
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
None,
"Respond bot",
600,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService()
tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService(session)
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session)
bot1_messages = [
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."},
]
async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio)
return all_audio
async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}")
if bot2_msg:
bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio)
return all_audio
async def argue():
for i in range(100):
print(f"In iteration {i}")
bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed"
(audio1, image_data1) = await asyncio.gather(
get_bot1_statement(), dalle.run_image_gen(bot1_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data1[1]),
AudioQueueFrame(audio1),
]
)
bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data2[1]),
AudioQueueFrame(audio2),
]
)
await asyncio.gather(transport.run(), argue())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

119
src/samples/foundational/08b-debate-generator.py
View File

@@ -1,119 +0,0 @@
import aiohttp
import argparse
import asyncio
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame
async def main(room_url:str):
async with aiohttp.ClientSession() as session:
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
None,
"Respond bot",
600,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = True
transport.camera_width = 1024
transport.camera_height = 1024
llm = AzureLLMService()
tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService(session)
dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session)
# dalle = OpenAIImageGenService(image_size="1024x1024")
topic = "Are pokemon edible?"
affirmative = "A woman dressed as a cowboy, outside on a ranch"
negative = "Pikachu in a business suit"
# topic = "Is a hot dog a sandwich?"
# affirmative = "A woman conservatively dressed as a librarian in a library surrounded by books"
# negative = "A cat dressed in a hot dog costume"
bot1_messages = [
{"role": "system", "content": f"You are {affirmative}. You're in a debate, and the topic is: '{topic}'. You're arguing the affirmative. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever agree with the user. Your responses should only be a few sentences long."},
]
bot2_messages = [
{"role": "system", "content": f"You are {negative}. You're in a debate, and the topic is: '{topic}'. You're arguing the negative. Debate this with the user, only responding with a few sentences. Don't ever agree with the user."},
]
async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}")
if bot1_msg:
bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio)
return all_audio
async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}")
if bot2_msg:
bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio)
return all_audio
async def argue():
for i in range(100):
print(f"In iteration {i}")
bot1_description = f"{affirmative}, cartoon, highly detailed"
(audio1, image_data1) = await asyncio.gather(
get_bot1_statement(), dalle.run_image_gen(bot1_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data1[1]),
AudioQueueFrame(audio1),
]
)
bot2_description = f"{negative}, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description)
)
await transport.send_queue.put(
[
ImageQueueFrame(None, image_data2[1]),
AudioQueueFrame(audio2),
]
)
await asyncio.gather(transport.run(), argue())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

56
src/samples/foundational/10-wake-word.py
View File

@@ -1,18 +1,15 @@
import aiohttp
import argparse
import asyncio
import os
import random
import requests
import time
import urllib.parse
from typing import AsyncGenerator
from PIL import Image
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.queue_aggregators import LLMContextAggregator
from dailyai.queue_aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator
from dailyai.queue_frame import (
QueueFrame,
TextQueueFrame,
@@ -22,7 +19,8 @@ from dailyai.queue_frame import (
)
from dailyai.services.ai_services import AIService
from typing import AsyncGenerator
from samples.foundational.support.runner import configure
sprites = {}
image_files = [
@@ -123,8 +121,8 @@ async def main(room_url: str, token):
transport.camera_width = 720
transport.camera_height = 1280
llm = AzureLLMService()
tts = ElevenLabsTTSService(session)
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl")
isa = ImageSyncAggregator()
@transport.event_handler("on_first_other_participant_joined")
@@ -136,11 +134,11 @@ async def main(room_url: str, token):
{"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."},
]
tma_in = LLMContextAggregator(
messages, "user", transport.my_participant_id
tma_in = LLMUserContextAggregator(
messages, transport.my_participant_id
)
tma_out = LLMContextAggregator(
messages, "assistant", transport.my_participant_id
tma_out = LLMAssistantContextAggregator(
messages, transport.my_participant_id
)
tf = TranscriptFilter(transport.my_participant_id)
ncf = NameCheckFilter(["Santa Cat", "Santa"])
@@ -169,35 +167,5 @@ async def main(room_url: str, token):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 24 hours in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60 * 24
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))
(url, token) = configure()
asyncio.run(main(url, token))

139
src/samples/foundational/11-sound-effects.py
View File

@@ -1,19 +1,19 @@
import argparse
import aiohttp
import asyncio
import logging
import os
import wave
import requests
import time
import urllib.parse
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.queue_aggregators import LLMContextAggregator, LLMUserContextAggregator, LLMAssistantContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator
from samples.foundational.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
@@ -64,52 +64,56 @@ class InboundSoundEffectWrapper(AIService):
async def main(room_url: str, token):
global transport
global llm
global tts
async with aiohttp.ClientSession() as session:
transport = DailyTransportService(
room_url,
token,
"Respond bot",
5,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
global transport
global llm
global tts
llm = AzureLLMService()
tts = AzureTTSService()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMUserContextAggregator(
messages, transport.my_participant_id
transport = DailyTransportService(
room_url,
token,
"Respond bot",
5,
)
tma_out = LLMAssistantContextAggregator(
messages, transport.my_participant_id
)
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper()
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue,
tts.run(
fl.run(
tma_out.run(
llm.run(
fl2.run(
in_sound.run(
tma_in.run(
transport.get_receive_frames()
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV")
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMUserContextAggregator(
messages, transport.my_participant_id
)
tma_out = LLMAssistantContextAggregator(
messages, transport.my_participant_id
)
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper()
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue,
tts.run(
fl.run(
tma_out.run(
llm.run(
fl2.run(
in_sound.run(
tma_in.run(
transport.get_receive_frames()
)
)
)
)
@@ -117,43 +121,12 @@ async def main(room_url: str, token):
)
)
)
)
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token))
(url, token) = configure()
asyncio.run(main(url, token))

165
src/samples/foundational/11a-dial-out.py
View File

@@ -1,165 +0,0 @@
import argparse
import asyncio
import os
import wave
import requests
import time
import urllib.parse
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_aggregators import LLMContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator
sounds = {}
sound_files = [
'ding1.wav',
'ding2.wav'
]
script_dir = os.path.dirname(__file__)
for file in sound_files:
# Build the full path to the image file
full_path = os.path.join(script_dir, "assets", file)
# Get the filename without the extension to use as the dictionary key
filename = os.path.splitext(os.path.basename(full_path))[0]
# Open the image and convert it to bytes
with wave.open(full_path) as audio_file:
sounds[file] = audio_file.readframes(-1)
class OutboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMResponseEndQueueFrame):
yield AudioQueueFrame(sounds["ding1.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
class InboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
yield AudioQueueFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
async def main(room_url: str, token, phone):
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
token,
"Respond bot",
300,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService()
tts = AzureTTSService()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMContextAggregator(
messages, "user", transport.my_participant_id
)
tma_out = LLMContextAggregator(
messages, "assistant", transport.my_participant_id
)
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper()
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue,
tts.run(
tma_out.run(
llm.run(
fl2.run(
in_sound.run(
tma_in.run(
transport.get_receive_frames()
)
)
)
)
)
)
)
@transport.event_handler("on_participant_joined")
async def pax_joined(transport, pax):
print(f"PARTICIPANT JOINED: {pax}")
@transport.event_handler("on_call_state_updated")
async def on_call_state_updated(transport, state):
if (state == "joined"):
if (phone):
transport.start_recording()
transport.dialout(phone)
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=True,
help="Daily API Key (needed to create token)",
)
parser.add_argument("-p", "--phone", type=str, required=False, help="A phone number to call when the bot joins the room")
args, unknown = parser.parse_known_args()
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(args.url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.staging.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {args.apikey}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
asyncio.run(main(args.url, token, args.phone))

11
src/samples/foundational/13-whisper-transcription.py
View File

@@ -1,9 +1,9 @@
import argparse
import asyncio
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.whisper_ai_services import WhisperSTTService
from samples.foundational.support.runner import configure
async def main(room_url: str):
global transport
@@ -35,10 +35,5 @@ async def main(room_url: str):
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))
(url, token) = configure()
asyncio.run(main(url))

52
src/samples/foundational/support/runner.py Normal file
View File

@@ -0,0 +1,52 @@
import argparse
import os
import time
import urllib
import requests
from dotenv import load_dotenv
load_dotenv()
def configure():
parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample")
parser.add_argument(
"-u", "--url", type=str, required=False, help="URL of the Daily room to join"
)
parser.add_argument(
"-k",
"--apikey",
type=str,
required=False,
help="Daily API Key (needed to create an owner token for the room)",
)
args, unknown = parser.parse_known_args()
url = args.url or os.getenv("DAILY_SAMPLE_ROOM_URL")
key = args.apikey or os.getenv("DAILY_API_KEY")
if not url:
raise Exception("No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.")
if not key:
raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.")
# Create a meeting token for the given room with an expiration 1 hour in the future.
room_name: str = urllib.parse.urlparse(url).path[1:]
expiration: float = time.time() + 60 * 60
res: requests.Response = requests.post(
f"https://api.daily.co/v1/meeting-tokens",
headers={"Authorization": f"Bearer {key}"},
json={
"properties": {"room_name": room_name, "is_owner": True, "exp": expiration}
},
)
if res.status_code != 200:
raise Exception(f"Failed to create meeting token: {res.status_code} {res.text}")
token: str = res.json()["token"]
return (url, token)

135
src/samples/internal/11a-dial-out.py Normal file
View File

@@ -0,0 +1,135 @@
import aiohttp
import asyncio
import os
import wave
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_aggregators import LLMContextAggregator
from dailyai.services.ai_services import AIService, FrameLogger
from dailyai.queue_frame import QueueFrame, AudioQueueFrame, LLMResponseEndQueueFrame, LLMMessagesQueueFrame
from typing import AsyncGenerator
from samples.foundational.support.runner import configure
sounds = {}
sound_files = [
'ding1.wav',
'ding2.wav'
]
script_dir = os.path.dirname(__file__)
for file in sound_files:
# Build the full path to the image file
full_path = os.path.join(script_dir, "assets", file)
# Get the filename without the extension to use as the dictionary key
filename = os.path.splitext(os.path.basename(full_path))[0]
# Open the image and convert it to bytes
with wave.open(full_path) as audio_file:
sounds[file] = audio_file.readframes(-1)
class OutboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMResponseEndQueueFrame):
yield AudioQueueFrame(sounds["ding1.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
class InboundSoundEffectWrapper(AIService):
def __init__(self):
pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, LLMMessagesQueueFrame):
yield AudioQueueFrame(sounds["ding2.wav"])
# In case anything else up the stack needs it
yield frame
else:
yield frame
async def main(room_url: str, token, phone):
async with aiohttp.ClientSession() as session:
global transport
global llm
global tts
transport = DailyTransportService(
room_url,
token,
"Respond bot",
300,
)
transport.mic_enabled = True
transport.mic_sample_rate = 16000
transport.camera_enabled = False
llm = AzureLLMService()
tts = AzureTTSService()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"]))
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
]
tma_in = LLMContextAggregator(
messages, "user", transport.my_participant_id
)
tma_out = LLMContextAggregator(
messages, "assistant", transport.my_participant_id
)
out_sound = OutboundSoundEffectWrapper()
in_sound = InboundSoundEffectWrapper()
fl = FrameLogger("LLM Out")
fl2 = FrameLogger("Transcription In")
await out_sound.run_to_queue(
transport.send_queue,
tts.run(
tma_out.run(
llm.run(
fl2.run(
in_sound.run(
tma_in.run(
transport.get_receive_frames()
)
)
)
)
)
)
)
@transport.event_handler("on_participant_joined")
async def pax_joined(transport, pax):
print(f"PARTICIPANT JOINED: {pax}")
@transport.event_handler("on_call_state_updated")
async def on_call_state_updated(transport, state):
if (state == "joined"):
if (phone):
transport.start_recording()
transport.dialout(phone)
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__":
(url, token) = configure()
asyncio.run(main(url, token))

4
src/samples/server/daily-bot-manager.py
View File

@@ -21,7 +21,7 @@ def start_bot(bot_path, args=None):
daily_api_key = os.getenv("DAILY_API_KEY")
api_path = os.getenv("DAILY_API_PATH") or "https://api.daily.co/v1"
timeout = int(os.getenv("ROOM_TIMEOUT") or os.getenv("BOT_MAX_DURATION") or 300)
timeout = int(os.getenv("DAILY_ROOM_TIMEOUT") or os.getenv("DAILY_BOT_MAX_DURATION") or 300)
exp = time.time() + timeout
res = requests.post(
f"{api_path}/rooms",
@@ -82,7 +82,7 @@ def start_bot(bot_path, args=None):
# Additional client config
config = {}
if os.getenv("CLIENT_VAD_TIMEOUT_SEC"):
config['vad_timeout_sec'] = float(os.getenv("CLIENT_VAD_TIMEOUT_SEC"))
config['vad_timeout_sec'] = float(os.getenv("DAILY_CLIENT_VAD_TIMEOUT_SEC"))
else:
config['vad_timeout_sec'] = 1.5