wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaned up example logging (#46)

Browse Source
This commit is contained in:
chadbailey59
2024-03-08 15:25:17 -06:00
committed by GitHub
parent 95a1efbe75
commit 8241dc0bed
20 changed files with 410 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
src/examples/foundational/01-say-one-thing.py
View File

@@ -1,5 +1,6 @@
import asyncio
import aiohttp
import logging
import os
from dailyai.services.daily_transport_service import DailyTransportService
@@ -8,6 +9,9 @@ from dailyai.services.playht_ai_service import PlayHTAIService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url):
async with aiohttp.ClientSession() as session:
@@ -21,24 +25,14 @@ async def main(room_url):
# to the daily-python basic API
meeting_duration_minutes = 5
transport = DailyTransportService(
room_url,
None,
"Say One Thing",
meeting_duration_minutes,
mic_enabled=True
room_url, None, "Say One Thing", meeting_duration_minutes, mic_enabled=True
)
tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
"""
tts = PlayHTAIService(
api_key=os.getenv("PLAY_HT_API_KEY"),
user_id=os.getenv("PLAY_HT_USER_ID"),
voice_url=os.getenv("PLAY_HT_VOICE_URL"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
"""
# Register an event handler so we can play the audio when the participant joins.
@transport.event_handler("on_participant_joined")
@@ -56,7 +50,7 @@ async def main(room_url):
await transport.stop_when_done()
await transport.run()
del(tts)
del tts
if __name__ == "__main__":

7
src/examples/foundational/01a-local-transport.py
View File

@@ -1,17 +1,20 @@
import asyncio
import aiohttp
import logging
import os
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.local_transport_service import LocalTransportService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main():
async with aiohttp.ClientSession() as session:
meeting_duration_minutes = 1
transport = LocalTransportService(
duration_minutes=meeting_duration_minutes,
mic_enabled=True
duration_minutes=meeting_duration_minutes, mic_enabled=True
)
tts = ElevenLabsTTSService(
aiohttp_session=session,

30
src/examples/foundational/02-llm-say-one-thing.py
View File

@@ -1,5 +1,6 @@
import asyncio
import os
import logging
import aiohttp
@@ -11,6 +12,9 @@ from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url):
async with aiohttp.ClientSession() as session:
@@ -20,25 +24,23 @@ async def main(room_url):
None,
"Say One Thing From an LLM",
duration_minutes=meeting_duration_minutes,
mic_enabled=True
mic_enabled=True,
)
tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
# tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE"))
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
# llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY"))
messages = [{
"role": "system",
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world."
}]
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
)
messages = [
{
"role": "system",
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
}
]
tts_task = asyncio.create_task(
tts.run_to_queue(
transport.send_queue,

14
src/examples/foundational/03-still-frame.py
View File

@@ -1,5 +1,6 @@
import asyncio
import aiohttp
import logging
import os
from dailyai.pipeline.frames import TextFrame
@@ -10,6 +11,9 @@ from dailyai.services.azure_ai_services import AzureImageGenServiceREST
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
local_joined = False
participant_joined = False
@@ -25,21 +29,23 @@ async def main(room_url):
mic_enabled=False,
camera_enabled=True,
camera_width=1024,
camera_height=1024
camera_height=1024,
)
imagegen = FalImageGenService(
image_size="1024x1024",
aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET"))
key_secret=os.getenv("FAL_KEY_SECRET"),
)
# imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
# imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
image_task = asyncio.create_task(
imagegen.run_to_queue(
transport.send_queue, [
TextFrame("a cat in the style of picasso")]))
transport.send_queue, [TextFrame("a cat in the style of picasso")]
)
)
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):

6
src/examples/foundational/03a-image-local.py
View File

@@ -1,5 +1,6 @@
import asyncio
import aiohttp
import logging
import os
import tkinter as tk
@@ -8,6 +9,10 @@ from dailyai.pipeline.frames import TextFrame
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.local_transport_service import LocalTransportService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
local_joined = False
participant_joined = False
@@ -46,5 +51,6 @@ async def main():
await asyncio.gather(transport.run(), image_task, run_tk())
if __name__ == "__main__":
asyncio.run(main())

26
src/examples/foundational/04-utterance-and-speech.py
View File

@@ -1,4 +1,5 @@
import asyncio
import logging
import os
import aiohttp
@@ -8,9 +9,12 @@ from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str):
async with aiohttp.ClientSession() as session:
@@ -21,20 +25,23 @@ async def main(room_url: str):
duration_minutes=1,
mic_enabled=True,
mic_sample_rate=16000,
camera_enabled=False
camera_enabled=False,
)
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
model=os.getenv("AZURE_CHATGPT_MODEL"),
)
azure_tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"))
region=os.getenv("AZURE_SPEECH_REGION"),
)
elevenlabs_tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
messages = [{"role": "system", "content": "tell the user a joke about llamas"}]
@@ -43,14 +50,19 @@ async def main(room_url: str):
# speak the LLM response.
buffer_queue = asyncio.Queue()
source_queue = asyncio.Queue()
pipeline = Pipeline(source = source_queue, sink=buffer_queue, processors=[llm, elevenlabs_tts])
pipeline = Pipeline(
source=source_queue, sink=buffer_queue, processors=[llm, elevenlabs_tts]
)
await source_queue.put(LLMMessagesQueueFrame(messages))
await source_queue.put(EndFrame())
pipeline_run_task = pipeline.run_pipeline()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await azure_tts.say("My friend the LLM is now going to tell a joke about llamas.", transport.send_queue)
await azure_tts.say(
"My friend the LLM is now going to tell a joke about llamas.",
transport.send_queue,
)
async def buffer_to_send_queue():
while True:

38
src/examples/foundational/05-sync-speech-and-image.py
View File

@@ -2,11 +2,27 @@ import asyncio
from re import S
import aiohttp
import os
from dailyai.pipeline.aggregators import GatedAggregator, LLMFullResponseAggregator, ParallelPipeline, SentenceAggregator
import logging
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, LLMResponseStartFrame
from dailyai.pipeline.aggregators import (
GatedAggregator,
LLMFullResponseAggregator,
ParallelPipeline,
SentenceAggregator,
)
from dailyai.pipeline.frames import (
AudioFrame,
EndFrame,
ImageFrame,
LLMMessagesQueueFrame,
LLMResponseStartFrame,
)
from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService
from dailyai.services.azure_ai_services import (
AzureLLMService,
AzureImageGenServiceREST,
AzureTTSService,
)
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService
@@ -14,6 +30,10 @@ from dailyai.services.open_ai_services import OpenAIImageGenService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url):
async with aiohttp.ClientSession() as session:
@@ -27,23 +47,26 @@ async def main(room_url):
camera_enabled=True,
mic_sample_rate=16000,
camera_width=1024,
camera_height=1024
camera_height=1024,
)
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="ErXwobaYiN019PkySvjV")
voice_id="ErXwobaYiN019PkySvjV",
)
dalle = FalImageGenService(
image_size="1024x1024",
aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET"))
key_secret=os.getenv("FAL_KEY_SECRET"),
)
source_queue = asyncio.Queue()
@@ -101,6 +124,7 @@ async def main(room_url):
await transport.run()
if __name__ == "__main__":
(url, token) = configure()
asyncio.run(main(url))

10
src/examples/foundational/05a-local-sync-speech-and-text.py
View File

@@ -1,6 +1,7 @@
import aiohttp
import argparse
import asyncio
import logging
import tkinter as tk
import os
@@ -10,6 +11,10 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.local_transport_service import LocalTransportService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url):
async with aiohttp.ClientSession() as session:
@@ -67,9 +72,7 @@ async def main(room_url):
to_speak = f"{month}: {image_description}"
audio_task = asyncio.create_task(get_all_audio(to_speak))
image_task = asyncio.create_task(dalle.run_image_gen(image_description))
(audio, image_data) = await asyncio.gather(
audio_task, image_task
)
(audio, image_data) = await asyncio.gather(audio_task, image_task)
return {
"month": month,
@@ -123,6 +126,7 @@ async def main(room_url):
await asyncio.gather(transport.run(), show_images(), run_tk())
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument(

116
src/examples/foundational/06-listen-and-respond.py
View File

@@ -1,65 +1,81 @@
import asyncio
import aiohttp
import logging
import os
from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.services.ai_services import FrameLogger
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator
from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMUserContextAggregator,
)
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str, token):
transport = DailyTransportService(
room_url,
token,
"Respond bot",
duration_minutes=5,
start_transcription=True,
mic_enabled=True,
mic_sample_rate=16000,
camera_enabled=False,
vad_enabled=True
)
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"))
fl = FrameLogger("Inner")
fl2 = FrameLogger("Outer")
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
async def handle_transcriptions():
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
]
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
tma_out = LLMAssistantContextAggregator(messages, transport._my_participant_id)
pipeline = Pipeline(
processors=[
fl,
tma_in,
llm,
fl2,
tts,
tma_out,
],
async with aiohttp.ClientSession() as session:
transport = DailyTransportService(
room_url,
token,
"Respond bot",
duration_minutes=5,
start_transcription=True,
mic_enabled=True,
mic_sample_rate=16000,
camera_enabled=False,
vad_enabled=True,
)
await transport.run_uninterruptible_pipeline(pipeline)
transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
)
fl = FrameLogger("Inner")
fl2 = FrameLogger("Outer")
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
async def handle_transcriptions():
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
]
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
tma_out = LLMAssistantContextAggregator(
messages, transport._my_participant_id
)
pipeline = Pipeline(
processors=[
fl,
tma_in,
llm,
fl2,
tts,
tma_out,
],
)
await transport.run_uninterruptible_pipeline(pipeline)
transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__":

42
src/examples/foundational/06a-image-sync.py
View File

@@ -1,23 +1,29 @@
import argparse
import asyncio
import os
import logging
from typing import AsyncGenerator
import aiohttp
import requests
import time
import urllib.parse
from PIL import Image
from dailyai.pipeline.frames import ImageFrame, Frame
from dailyai.pipeline.frames import ImageFrame, Frame
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.ai_services import AIService
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator
from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMUserContextAggregator,
)
from dailyai.services.fal_ai_services import FalImageGenService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
class ImageSyncAggregator(AIService):
def __init__(self, speaking_path: str, waiting_path: str):
@@ -50,15 +56,18 @@ async def main(room_url: str, token):
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"))
region=os.getenv("AZURE_SPEECH_REGION"),
)
img = FalImageGenService(
image_size="1024x1024",
aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET"))
key_secret=os.getenv("FAL_KEY_SECRET"),
)
async def get_images():
get_speaking_task = asyncio.create_task(
@@ -80,12 +89,13 @@ async def main(room_url: str, token):
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
]
tma_in = LLMUserContextAggregator(
messages, transport._my_participant_id
)
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
tma_out = LLMAssistantContextAggregator(
messages, transport._my_participant_id
)
@@ -96,14 +106,8 @@ async def main(room_url: str, token):
await tts.run_to_queue(
transport.send_queue,
image_sync_aggregator.run(
tma_out.run(
llm.run(
tma_in.run(
transport.get_receive_frames()
)
)
)
)
tma_out.run(llm.run(tma_in.run(transport.get_receive_frames())))
),
)
transport.transcription_settings["extra"]["punctuate"] = True

32
src/examples/foundational/07-interruptible.py
View File

@@ -1,14 +1,23 @@
import asyncio
import aiohttp
import logging
import os
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMResponseAggregator, LLMUserContextAggregator, UserResponseAggregator
from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMResponseAggregator,
LLMUserContextAggregator,
UserResponseAggregator,
)
from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.ai_services import FrameLogger
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from examples.support.runner import configure
from support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str, token):
@@ -28,10 +37,12 @@ async def main(room_url: str, token):
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"))
region=os.getenv("AZURE_SPEECH_REGION"),
)
pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
@@ -41,17 +52,16 @@ async def main(room_url: str, token):
async def run_conversation():
messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
]
await transport.run_interruptible_pipeline(
pipeline,
post_processor=LLMResponseAggregator(
messages
),
pre_processor=UserResponseAggregator(
messages
),
post_processor=LLMResponseAggregator(messages),
pre_processor=UserResponseAggregator(messages),
)
transport.transcription_settings["extra"]["punctuate"] = False

28
src/examples/foundational/08-bots-arguing.py
View File

@@ -1,5 +1,6 @@
import aiohttp
import asyncio
import logging
import os
from dailyai.services.daily_transport_service import DailyTransportService
@@ -7,9 +8,12 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.pipeline.frames import AudioFrame, ImageFrame
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str):
async with aiohttp.ClientSession() as session:
@@ -22,33 +26,41 @@ async def main(room_url: str):
mic_sample_rate=16000,
camera_enabled=True,
camera_width=1024,
camera_height=1024
camera_height=1024,
)
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts1 = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"))
region=os.getenv("AZURE_SPEECH_REGION"),
)
tts2 = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="jBpfuIE2acCO8z3wKNLl")
voice_id="jBpfuIE2acCO8z3wKNLl",
)
dalle = FalImageGenService(
image_size="1024x1024",
aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET"))
key_secret=os.getenv("FAL_KEY_SECRET"),
)
bot1_messages = [
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."},
{
"role": "system",
"content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long.",
},
]
bot2_messages = [
{
"role": "system",
"content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."},
"content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich.",
},
]
async def get_bot1_statement():

68
src/examples/foundational/10-wake-word.py
View File

@@ -1,15 +1,18 @@
import aiohttp
import asyncio
import logging
import os
import random
from typing import AsyncGenerator
from PIL import Image
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.pipeline.aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator
from dailyai.pipeline.aggregators import (
LLMUserContextAggregator,
LLMAssistantContextAggregator,
)
from dailyai.pipeline.frames import (
Frame,
TextFrame,
@@ -18,19 +21,21 @@ from dailyai.pipeline.frames import (
TranscriptionQueueFrame,
)
from dailyai.services.ai_services import AIService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
sprites = {}
image_files = [
'sc-default.png',
'sc-talk.png',
'sc-listen-1.png',
'sc-think-1.png',
'sc-think-2.png',
'sc-think-3.png',
'sc-think-4.png'
"sc-default.png",
"sc-talk.png",
"sc-listen-1.png",
"sc-think-1.png",
"sc-think-2.png",
"sc-think-3.png",
"sc-think-4.png",
]
script_dir = os.path.dirname(__file__)
@@ -47,16 +52,17 @@ for file in image_files:
# When the bot isn't talking, show a static image of the cat listening
quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
# When the bot is talking, build an animation from two sprites
talking_list = [sprites['sc-default.png'], sprites['sc-talk.png']]
talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
talking = [random.choice(talking_list) for x in range(30)]
talking_frame = SpriteFrame(images=talking)
# TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing
thinking_list = [
sprites['sc-think-1.png'],
sprites['sc-think-2.png'],
sprites['sc-think-3.png'],
sprites['sc-think-4.png']]
sprites["sc-think-1.png"],
sprites["sc-think-2.png"],
sprites["sc-think-3.png"],
sprites["sc-think-4.png"],
]
thinking_frame = SpriteFrame(images=thinking_list)
@@ -115,7 +121,7 @@ async def main(room_url: str, token):
mic_sample_rate=16000,
camera_enabled=True,
camera_width=720,
camera_height=1280
camera_height=1280,
)
transport._mic_enabled = True
transport._mic_sample_rate = 16000
@@ -126,25 +132,31 @@ async def main(room_url: str, token):
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="jBpfuIE2acCO8z3wKNLl")
voice_id="jBpfuIE2acCO8z3wKNLl",
)
isa = ImageSyncAggregator()
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi! If you want to talk to me, just say 'hey Santa Cat'.", transport.send_queue)
await tts.say(
"Hi! If you want to talk to me, just say 'hey Santa Cat'.",
transport.send_queue,
)
async def handle_transcriptions():
messages = [
{"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."},
{
"role": "system",
"content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long.",
},
]
tma_in = LLMUserContextAggregator(
messages, transport._my_participant_id
)
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
tma_out = LLMAssistantContextAggregator(
messages, transport._my_participant_id
)
@@ -155,16 +167,10 @@ async def main(room_url: str, token):
isa.run(
tma_out.run(
llm.run(
tma_in.run(
ncf.run(
tf.run(
transport.get_receive_frames()
)
)
)
tma_in.run(ncf.run(tf.run(transport.get_receive_frames())))
)
)
)
),
)
async def starting_image():

2
src/examples/foundational/11-sound-effects.py
View File

@@ -14,7 +14,7 @@ from typing import AsyncGenerator
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)

12
src/examples/foundational/13-whisper-transcription.py
View File

@@ -1,10 +1,14 @@
import asyncio
import logging
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.whisper_ai_services import WhisperSTTService
from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str):
transport = DailyTransportService(
@@ -14,7 +18,7 @@ async def main(room_url: str):
start_transcription=True,
mic_enabled=False,
camera_enabled=False,
speaker_enabled=True
speaker_enabled=True,
)
stt = WhisperSTTService()
@@ -28,9 +32,9 @@ async def main(room_url: str):
async def handle_speaker():
await stt.run_to_queue(
transcription_output_queue,
transport.get_receive_frames()
transcription_output_queue, transport.get_receive_frames()
)
await asyncio.gather(transport.run(), handle_speaker(), handle_transcription())

7
src/examples/foundational/13a-whisper-local.py
View File

@@ -1,11 +1,16 @@
import argparse
import asyncio
import logging
import wave
from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
from dailyai.services.local_transport_service import LocalTransportService
from dailyai.services.whisper_ai_services import WhisperSTTService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str):
global transport
@@ -17,7 +22,7 @@ async def main(room_url: str):
camera_enabled=False,
speaker_enabled=True,
duration_minutes=meeting_duration_minutes,
start_transcription=True
start_transcription=True,
)
stt = WhisperSTTService()
transcription_output_queue = asyncio.Queue()

0
src/examples/foundational/assets/clack-short-quiet.wav → src/examples/starter-apps/assets/clack-short-quiet.wav
View File

0
src/examples/foundational/assets/clack-short.wav → src/examples/starter-apps/assets/clack-short.wav
View File

0
src/examples/foundational/assets/clack.wav → src/examples/starter-apps/assets/clack.wav
View File

226
src/examples/starter-apps/patient-intake.py
View File

@@ -2,35 +2,48 @@ import aiohttp
import asyncio
import json
import random
import logging
import os
import re
import wave
from typing import AsyncGenerator
from PIL import Image
import sys
print('\n'.join(sys.path))
from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator, UserResponseAggregator, LLMResponseAggregator
from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMContextAggregator,
LLMUserContextAggregator,
UserResponseAggregator,
LLMResponseAggregator,
)
from examples.support.runner import configure
from dailyai.pipeline.frames import LLMMessagesQueueFrame, TranscriptionQueueFrame, Frame, TextFrame, LLMFunctionCallFrame, LLMFunctionStartFrame, LLMResponseEndFrame, StartFrame, AudioFrame, SpriteFrame, ImageFrame
from dailyai.pipeline.frames import (
LLMMessagesQueueFrame,
TranscriptionQueueFrame,
Frame,
TextFrame,
LLMFunctionCallFrame,
LLMFunctionStartFrame,
LLMResponseEndFrame,
StartFrame,
AudioFrame,
SpriteFrame,
ImageFrame,
)
from dailyai.services.ai_services import FrameLogger, AIService
import logging
logging.basicConfig(level=logging.INFO)
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
sounds = {}
sound_files = [
'clack-short.wav',
'clack.wav',
'clack-short-quiet.wav'
]
sound_files = ["clack-short.wav", "clack.wav", "clack-short-quiet.wav"]
script_dir = os.path.dirname(__file__)
@@ -48,9 +61,11 @@ steps = [
{
"prompt": "Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function.",
"run_async": False,
"failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.", "tools": [{
"type": "function",
"function": {
"failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.",
"tools": [
{
"type": "function",
"function": {
"name": "verify_birthday",
"description": "Use this function to verify the user has provided their correct birthday.",
"parameters": {
@@ -58,18 +73,21 @@ steps = [
"properties": {
"birthday": {
"type": "string",
"description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function."
"description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function.",
}
}
}
},
},
},
}
}]},
],
},
{
"prompt": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages.",
"run_async": True,
"tools": [{
"type": "function",
"function": {
"tools": [
{
"type": "function",
"function": {
"name": "list_prescriptions",
"description": "Once the user has provided a list of their prescription medications, call this function.",
"parameters": {
@@ -82,19 +100,20 @@ steps = [
"properties": {
"medication": {
"type": "string",
"description": "The medication's name"
"description": "The medication's name",
},
"dosage": {
"type": "string",
"description": "The prescription's dosage"
}
}
}
"description": "The prescription's dosage",
},
},
},
}
}
}
},
},
},
}
}]
],
},
{
"prompt": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function.",
@@ -115,16 +134,16 @@ steps = [
"properties": {
"name": {
"type": "string",
"description": "What the user is allergic to"
"description": "What the user is allergic to",
}
}
}
},
},
}
}
}
}
},
},
},
}
]
],
},
{
"prompt": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function.",
@@ -145,14 +164,14 @@ steps = [
"properties": {
"name": {
"type": "string",
"description": "The user's medical condition"
"description": "The user's medical condition",
}
}
}
},
},
}
}
}
}
},
},
},
},
],
},
@@ -175,20 +194,23 @@ steps = [
"properties": {
"name": {
"type": "string",
"description": "The user's reason for visiting the doctor"
"description": "The user's reason for visiting the doctor",
}
}
}
},
},
}
}
}
}
},
},
},
}
]
],
},
{"prompt": "Now, thank the user and end the conversation.",
"run_async": True, "tools": []},
{"prompt": "", "run_async": True, "tools": []}
{
"prompt": "Now, thank the user and end the conversation.",
"run_async": True,
"tools": [],
},
{"prompt": "", "run_async": True, "tools": []},
]
current_step = 0
@@ -219,15 +241,15 @@ class ChecklistProcessor(AIService):
"list_prescriptions",
"list_allergies",
"list_conditions",
"list_visit_reasons"
"list_visit_reasons",
]
messages.append(
{"role": "system", "content": f"{self._id} {steps[0]['prompt']}"})
{"role": "system", "content": f"{self._id} {steps[0]['prompt']}"}
)
def verify_birthday(self, args):
return args['birthday'] == "1983-01-01"
return args["birthday"] == "1983-01-01"
def list_prescriptions(self, args):
# print(f"--- Prescriptions: {args['prescriptions']}\n")
@@ -250,18 +272,21 @@ class ChecklistProcessor(AIService):
this_step = steps[current_step]
# TODO-CB: forcing a global here :/
self._tools.clear()
self._tools.extend(this_step['tools'])
self._tools.extend(this_step["tools"])
if isinstance(frame, LLMFunctionStartFrame):
print(f"... Preparing function call: {frame.function_name}")
self._function_name = frame.function_name
if this_step['run_async']:
if this_step["run_async"]:
# Get the LLM talking about the next step before getting the rest
# of the function call completion
current_step += 1
self._messages.append({
"role": "system", "content": steps[current_step]['prompt']})
self._messages.append(
{"role": "system", "content": steps[current_step]["prompt"]}
)
yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame
else:
# Insert a quick response while we run the function
@@ -270,29 +295,37 @@ class ChecklistProcessor(AIService):
elif isinstance(frame, LLMFunctionCallFrame):
if frame.function_name and frame.arguments:
print(
f"--> Calling function: {frame.function_name} with arguments:")
pretty_json = re.sub("\n", "\n ", json.dumps(
json.loads(frame.arguments), indent=2))
print(f"--> Calling function: {frame.function_name} with arguments:")
pretty_json = re.sub(
"\n", "\n ", json.dumps(json.loads(frame.arguments), indent=2)
)
print(f"--> {pretty_json}\n")
if not frame.function_name in self._functions:
raise Exception(f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions.")
raise Exception(
f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions."
)
fn = getattr(self, frame.function_name)
result = fn(json.loads(frame.arguments))
if not this_step['run_async']:
if not this_step["run_async"]:
if result:
current_step += 1
self._messages.append({
"role": "system", "content": steps[current_step]['prompt']})
self._messages.append(
{"role": "system", "content": steps[current_step]["prompt"]}
)
yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame
else:
self._messages.append({
"role": "system", "content": this_step['failed']})
self._messages.append(
{"role": "system", "content": this_step["failed"]}
)
yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame
print(f"<-- Verify result: {result}\n")
@@ -315,7 +348,7 @@ async def main(room_url: str, token):
mic_sample_rate=16000,
camera_enabled=False,
start_transcription=True,
vad_enabled=True
vad_enabled=True,
)
# TODO-CB: Go back to vad_enabled
@@ -324,12 +357,18 @@ async def main(room_url: str, token):
# llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv(
# "AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
llm = OpenAILLMService(api_key=os.getenv(
"OPENAI_CHATGPT_API_KEY"), model="gpt-4-1106-preview", tools=tools) # gpt-4-1106-preview
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
model="gpt-4-1106-preview",
tools=tools,
) # gpt-4-1106-preview
# tts = AzureTTSService(api_key=os.getenv(
# "AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv(
"ELEVENLABS_API_KEY"), voice_id="XrExE9yKIg1WjnnlVkGX") # matilda
tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="XrExE9yKIg1WjnnlVkGX",
) # matilda
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv(
# "DEEPGRAM_API_KEY"), voice="aura-asteria-en")
@@ -345,34 +384,23 @@ async def main(room_url: str, token):
# TODO-CB: Make sure this message gets into the context somehow
await tts.run_to_queue(
transport.send_queue,
llm.run([LLMMessagesQueueFrame(messages)]),
llm.run([LLMMessagesQueueFrame(messages)]),
)
async def handle_intake():
pipeline = Pipeline(
processors=[
fl,
llm,
fl2,
checklist,
tts
]
pipeline = Pipeline(processors=[fl, llm, fl2, checklist, tts])
await transport.run_interruptible_pipeline(
pipeline,
post_processor=LLMResponseAggregator(messages),
pre_processor=UserResponseAggregator(messages),
)
await transport.run_interruptible_pipeline(pipeline,
post_processor=LLMResponseAggregator(
messages
),
pre_processor=UserResponseAggregator(messages)
)
transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True
try:
await asyncio.gather(transport.run(), handle_intake())
except (asyncio.CancelledError, KeyboardInterrupt):
print('whoops')
print("whoops")
transport.stop()