cleaned up example logging (#46)

This commit is contained in:
chadbailey59
2024-03-08 15:25:17 -06:00
committed by GitHub
parent 95a1efbe75
commit 8241dc0bed
20 changed files with 410 additions and 274 deletions

View File

@@ -1,5 +1,6 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
@@ -8,6 +9,9 @@ from dailyai.services.playht_ai_service import PlayHTAIService
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url): async def main(room_url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -21,24 +25,14 @@ async def main(room_url):
# to the daily-python basic API # to the daily-python basic API
meeting_duration_minutes = 5 meeting_duration_minutes = 5
transport = DailyTransportService( transport = DailyTransportService(
room_url, room_url, None, "Say One Thing", meeting_duration_minutes, mic_enabled=True
None,
"Say One Thing",
meeting_duration_minutes,
mic_enabled=True
) )
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID")) voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
"""
tts = PlayHTAIService(
api_key=os.getenv("PLAY_HT_API_KEY"),
user_id=os.getenv("PLAY_HT_USER_ID"),
voice_url=os.getenv("PLAY_HT_VOICE_URL"),
) )
"""
# Register an event handler so we can play the audio when the participant joins. # Register an event handler so we can play the audio when the participant joins.
@transport.event_handler("on_participant_joined") @transport.event_handler("on_participant_joined")
@@ -56,7 +50,7 @@ async def main(room_url):
await transport.stop_when_done() await transport.stop_when_done()
await transport.run() await transport.run()
del(tts) del tts
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -1,17 +1,20 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.local_transport_service import LocalTransportService from dailyai.services.local_transport_service import LocalTransportService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(): async def main():
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
meeting_duration_minutes = 1 meeting_duration_minutes = 1
transport = LocalTransportService( transport = LocalTransportService(
duration_minutes=meeting_duration_minutes, duration_minutes=meeting_duration_minutes, mic_enabled=True
mic_enabled=True
) )
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,

View File

@@ -1,5 +1,6 @@
import asyncio import asyncio
import os import os
import logging
import aiohttp import aiohttp
@@ -11,6 +12,9 @@ from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.open_ai_services import OpenAILLMService from dailyai.services.open_ai_services import OpenAILLMService
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url): async def main(room_url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -20,25 +24,23 @@ async def main(room_url):
None, None,
"Say One Thing From an LLM", "Say One Thing From an LLM",
duration_minutes=meeting_duration_minutes, duration_minutes=meeting_duration_minutes,
mic_enabled=True mic_enabled=True,
) )
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID")) voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
# tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) )
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE")) llm = OpenAILLMService(
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
llm = AzureLLMService( )
api_key=os.getenv("AZURE_CHATGPT_API_KEY"), messages = [
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), {
model=os.getenv("AZURE_CHATGPT_MODEL")) "role": "system",
# llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY")) "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
messages = [{ }
"role": "system", ]
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world."
}]
tts_task = asyncio.create_task( tts_task = asyncio.create_task(
tts.run_to_queue( tts.run_to_queue(
transport.send_queue, transport.send_queue,

View File

@@ -1,5 +1,6 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
from dailyai.pipeline.frames import TextFrame from dailyai.pipeline.frames import TextFrame
@@ -10,6 +11,9 @@ from dailyai.services.azure_ai_services import AzureImageGenServiceREST
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
local_joined = False local_joined = False
participant_joined = False participant_joined = False
@@ -25,21 +29,23 @@ async def main(room_url):
mic_enabled=False, mic_enabled=False,
camera_enabled=True, camera_enabled=True,
camera_width=1024, camera_width=1024,
camera_height=1024 camera_height=1024,
) )
imagegen = FalImageGenService( imagegen = FalImageGenService(
image_size="1024x1024", image_size="1024x1024",
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET")) key_secret=os.getenv("FAL_KEY_SECRET"),
)
# imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024") # imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
# imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL")) # imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
image_task = asyncio.create_task( image_task = asyncio.create_task(
imagegen.run_to_queue( imagegen.run_to_queue(
transport.send_queue, [ transport.send_queue, [TextFrame("a cat in the style of picasso")]
TextFrame("a cat in the style of picasso")])) )
)
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):

View File

@@ -1,5 +1,6 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
import tkinter as tk import tkinter as tk
@@ -8,6 +9,10 @@ from dailyai.pipeline.frames import TextFrame
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.local_transport_service import LocalTransportService from dailyai.services.local_transport_service import LocalTransportService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
local_joined = False local_joined = False
participant_joined = False participant_joined = False
@@ -46,5 +51,6 @@ async def main():
await asyncio.gather(transport.run(), image_task, run_tk()) await asyncio.gather(transport.run(), image_task, run_tk())
if __name__ == "__main__": if __name__ == "__main__":
asyncio.run(main()) asyncio.run(main())

View File

@@ -1,4 +1,5 @@
import asyncio import asyncio
import logging
import os import os
import aiohttp import aiohttp
@@ -8,9 +9,12 @@ from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str): async def main(room_url: str):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -21,20 +25,23 @@ async def main(room_url: str):
duration_minutes=1, duration_minutes=1,
mic_enabled=True, mic_enabled=True,
mic_sample_rate=16000, mic_sample_rate=16000,
camera_enabled=False camera_enabled=False,
) )
llm = AzureLLMService( llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"), api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL")) model=os.getenv("AZURE_CHATGPT_MODEL"),
)
azure_tts = AzureTTSService( azure_tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"), api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION")) region=os.getenv("AZURE_SPEECH_REGION"),
)
elevenlabs_tts = ElevenLabsTTSService( elevenlabs_tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID")) voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
messages = [{"role": "system", "content": "tell the user a joke about llamas"}] messages = [{"role": "system", "content": "tell the user a joke about llamas"}]
@@ -43,14 +50,19 @@ async def main(room_url: str):
# speak the LLM response. # speak the LLM response.
buffer_queue = asyncio.Queue() buffer_queue = asyncio.Queue()
source_queue = asyncio.Queue() source_queue = asyncio.Queue()
pipeline = Pipeline(source = source_queue, sink=buffer_queue, processors=[llm, elevenlabs_tts]) pipeline = Pipeline(
source=source_queue, sink=buffer_queue, processors=[llm, elevenlabs_tts]
)
await source_queue.put(LLMMessagesQueueFrame(messages)) await source_queue.put(LLMMessagesQueueFrame(messages))
await source_queue.put(EndFrame()) await source_queue.put(EndFrame())
pipeline_run_task = pipeline.run_pipeline() pipeline_run_task = pipeline.run_pipeline()
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
await azure_tts.say("My friend the LLM is now going to tell a joke about llamas.", transport.send_queue) await azure_tts.say(
"My friend the LLM is now going to tell a joke about llamas.",
transport.send_queue,
)
async def buffer_to_send_queue(): async def buffer_to_send_queue():
while True: while True:

View File

@@ -2,11 +2,27 @@ import asyncio
from re import S from re import S
import aiohttp import aiohttp
import os import os
from dailyai.pipeline.aggregators import GatedAggregator, LLMFullResponseAggregator, ParallelPipeline, SentenceAggregator import logging
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, LLMResponseStartFrame from dailyai.pipeline.aggregators import (
GatedAggregator,
LLMFullResponseAggregator,
ParallelPipeline,
SentenceAggregator,
)
from dailyai.pipeline.frames import (
AudioFrame,
EndFrame,
ImageFrame,
LLMMessagesQueueFrame,
LLMResponseStartFrame,
)
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService from dailyai.services.azure_ai_services import (
AzureLLMService,
AzureImageGenServiceREST,
AzureTTSService,
)
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
@@ -14,6 +30,10 @@ from dailyai.services.open_ai_services import OpenAIImageGenService
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url): async def main(room_url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -27,23 +47,26 @@ async def main(room_url):
camera_enabled=True, camera_enabled=True,
mic_sample_rate=16000, mic_sample_rate=16000,
camera_width=1024, camera_width=1024,
camera_height=1024 camera_height=1024,
) )
llm = AzureLLMService( llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"), api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL")) model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="ErXwobaYiN019PkySvjV") voice_id="ErXwobaYiN019PkySvjV",
)
dalle = FalImageGenService( dalle = FalImageGenService(
image_size="1024x1024", image_size="1024x1024",
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET")) key_secret=os.getenv("FAL_KEY_SECRET"),
)
source_queue = asyncio.Queue() source_queue = asyncio.Queue()
@@ -101,6 +124,7 @@ async def main(room_url):
await transport.run() await transport.run()
if __name__ == "__main__": if __name__ == "__main__":
(url, token) = configure() (url, token) = configure()
asyncio.run(main(url)) asyncio.run(main(url))

View File

@@ -1,6 +1,7 @@
import aiohttp import aiohttp
import argparse import argparse
import asyncio import asyncio
import logging
import tkinter as tk import tkinter as tk
import os import os
@@ -10,6 +11,10 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.local_transport_service import LocalTransportService from dailyai.services.local_transport_service import LocalTransportService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url): async def main(room_url):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -67,9 +72,7 @@ async def main(room_url):
to_speak = f"{month}: {image_description}" to_speak = f"{month}: {image_description}"
audio_task = asyncio.create_task(get_all_audio(to_speak)) audio_task = asyncio.create_task(get_all_audio(to_speak))
image_task = asyncio.create_task(dalle.run_image_gen(image_description)) image_task = asyncio.create_task(dalle.run_image_gen(image_description))
(audio, image_data) = await asyncio.gather( (audio, image_data) = await asyncio.gather(audio_task, image_task)
audio_task, image_task
)
return { return {
"month": month, "month": month,
@@ -123,6 +126,7 @@ async def main(room_url):
await asyncio.gather(transport.run(), show_images(), run_tk()) await asyncio.gather(transport.run(), show_images(), run_tk())
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument( parser.add_argument(

View File

@@ -1,65 +1,81 @@
import asyncio import asyncio
import aiohttp
import logging
import os import os
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.services.ai_services import FrameLogger from dailyai.services.ai_services import FrameLogger
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMUserContextAggregator,
)
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str, token): async def main(room_url: str, token):
transport = DailyTransportService( async with aiohttp.ClientSession() as session:
room_url, transport = DailyTransportService(
token, room_url,
"Respond bot", token,
duration_minutes=5, "Respond bot",
start_transcription=True, duration_minutes=5,
mic_enabled=True, start_transcription=True,
mic_sample_rate=16000, mic_enabled=True,
camera_enabled=False, mic_sample_rate=16000,
vad_enabled=True camera_enabled=False,
) vad_enabled=True,
llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL"))
tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION"))
fl = FrameLogger("Inner")
fl2 = FrameLogger("Outer")
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
async def handle_transcriptions():
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
]
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
tma_out = LLMAssistantContextAggregator(messages, transport._my_participant_id)
pipeline = Pipeline(
processors=[
fl,
tma_in,
llm,
fl2,
tts,
tma_out,
],
) )
await transport.run_uninterruptible_pipeline(pipeline)
transport.transcription_settings["extra"]["endpointing"] = True tts = ElevenLabsTTSService(
transport.transcription_settings["extra"]["punctuate"] = True aiohttp_session=session,
await asyncio.gather(transport.run(), handle_transcriptions()) api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
)
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
)
fl = FrameLogger("Inner")
fl2 = FrameLogger("Outer")
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await tts.say("Hi, I'm listening!", transport.send_queue)
async def handle_transcriptions():
messages = [
{
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
]
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
tma_out = LLMAssistantContextAggregator(
messages, transport._my_participant_id
)
pipeline = Pipeline(
processors=[
fl,
tma_in,
llm,
fl2,
tts,
tma_out,
],
)
await transport.run_uninterruptible_pipeline(pipeline)
transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True
await asyncio.gather(transport.run(), handle_transcriptions())
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -1,23 +1,29 @@
import argparse import argparse
import asyncio import asyncio
import os import os
import logging
from typing import AsyncGenerator from typing import AsyncGenerator
import aiohttp import aiohttp
import requests import requests
import time import time
import urllib.parse import urllib.parse
from PIL import Image from PIL import Image
from dailyai.pipeline.frames import ImageFrame, Frame
from dailyai.pipeline.frames import ImageFrame, Frame
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.ai_services import AIService from dailyai.services.ai_services import AIService
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMUserContextAggregator,
)
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
class ImageSyncAggregator(AIService): class ImageSyncAggregator(AIService):
def __init__(self, speaking_path: str, waiting_path: str): def __init__(self, speaking_path: str, waiting_path: str):
@@ -50,15 +56,18 @@ async def main(room_url: str, token):
llm = AzureLLMService( llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"), api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL")) model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = AzureTTSService( tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"), api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION")) region=os.getenv("AZURE_SPEECH_REGION"),
)
img = FalImageGenService( img = FalImageGenService(
image_size="1024x1024", image_size="1024x1024",
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET")) key_secret=os.getenv("FAL_KEY_SECRET"),
)
async def get_images(): async def get_images():
get_speaking_task = asyncio.create_task( get_speaking_task = asyncio.create_task(
@@ -80,12 +89,13 @@ async def main(room_url: str, token):
async def handle_transcriptions(): async def handle_transcriptions():
messages = [ messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, {
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
] ]
tma_in = LLMUserContextAggregator( tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
messages, transport._my_participant_id
)
tma_out = LLMAssistantContextAggregator( tma_out = LLMAssistantContextAggregator(
messages, transport._my_participant_id messages, transport._my_participant_id
) )
@@ -96,14 +106,8 @@ async def main(room_url: str, token):
await tts.run_to_queue( await tts.run_to_queue(
transport.send_queue, transport.send_queue,
image_sync_aggregator.run( image_sync_aggregator.run(
tma_out.run( tma_out.run(llm.run(tma_in.run(transport.get_receive_frames())))
llm.run( ),
tma_in.run(
transport.get_receive_frames()
)
)
)
)
) )
transport.transcription_settings["extra"]["punctuate"] = True transport.transcription_settings["extra"]["punctuate"] = True

View File

@@ -1,14 +1,23 @@
import asyncio import asyncio
import aiohttp import aiohttp
import logging
import os import os
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMResponseAggregator, LLMUserContextAggregator, UserResponseAggregator from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMResponseAggregator,
LLMUserContextAggregator,
UserResponseAggregator,
)
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.ai_services import FrameLogger from dailyai.services.ai_services import FrameLogger
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from examples.support.runner import configure
from support.runner import configure logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str, token): async def main(room_url: str, token):
@@ -28,10 +37,12 @@ async def main(room_url: str, token):
llm = AzureLLMService( llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"), api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL")) model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = AzureTTSService( tts = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"), api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION")) region=os.getenv("AZURE_SPEECH_REGION"),
)
pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts]) pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
@@ -41,17 +52,16 @@ async def main(room_url: str, token):
async def run_conversation(): async def run_conversation():
messages = [ messages = [
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, {
"role": "system",
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
},
] ]
await transport.run_interruptible_pipeline( await transport.run_interruptible_pipeline(
pipeline, pipeline,
post_processor=LLMResponseAggregator( post_processor=LLMResponseAggregator(messages),
messages pre_processor=UserResponseAggregator(messages),
),
pre_processor=UserResponseAggregator(
messages
),
) )
transport.transcription_settings["extra"]["punctuate"] = False transport.transcription_settings["extra"]["punctuate"] = False

View File

@@ -1,5 +1,6 @@
import aiohttp import aiohttp
import asyncio import asyncio
import logging
import os import os
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
@@ -7,9 +8,12 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.pipeline.frames import AudioFrame, ImageFrame from dailyai.pipeline.frames import AudioFrame, ImageFrame
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str): async def main(room_url: str):
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
@@ -22,33 +26,41 @@ async def main(room_url: str):
mic_sample_rate=16000, mic_sample_rate=16000,
camera_enabled=True, camera_enabled=True,
camera_width=1024, camera_width=1024,
camera_height=1024 camera_height=1024,
) )
llm = AzureLLMService( llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"), api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL")) model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts1 = AzureTTSService( tts1 = AzureTTSService(
api_key=os.getenv("AZURE_SPEECH_API_KEY"), api_key=os.getenv("AZURE_SPEECH_API_KEY"),
region=os.getenv("AZURE_SPEECH_REGION")) region=os.getenv("AZURE_SPEECH_REGION"),
)
tts2 = ElevenLabsTTSService( tts2 = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="jBpfuIE2acCO8z3wKNLl") voice_id="jBpfuIE2acCO8z3wKNLl",
)
dalle = FalImageGenService( dalle = FalImageGenService(
image_size="1024x1024", image_size="1024x1024",
aiohttp_session=session, aiohttp_session=session,
key_id=os.getenv("FAL_KEY_ID"), key_id=os.getenv("FAL_KEY_ID"),
key_secret=os.getenv("FAL_KEY_SECRET")) key_secret=os.getenv("FAL_KEY_SECRET"),
)
bot1_messages = [ bot1_messages = [
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."}, {
"role": "system",
"content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long.",
},
] ]
bot2_messages = [ bot2_messages = [
{ {
"role": "system", "role": "system",
"content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich.",
},
] ]
async def get_bot1_statement(): async def get_bot1_statement():

View File

@@ -1,15 +1,18 @@
import aiohttp import aiohttp
import asyncio import asyncio
import logging
import os import os
import random import random
from typing import AsyncGenerator from typing import AsyncGenerator
from PIL import Image from PIL import Image
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.pipeline.aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator from dailyai.pipeline.aggregators import (
LLMUserContextAggregator,
LLMAssistantContextAggregator,
)
from dailyai.pipeline.frames import ( from dailyai.pipeline.frames import (
Frame, Frame,
TextFrame, TextFrame,
@@ -18,19 +21,21 @@ from dailyai.pipeline.frames import (
TranscriptionQueueFrame, TranscriptionQueueFrame,
) )
from dailyai.services.ai_services import AIService from dailyai.services.ai_services import AIService
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
sprites = {} sprites = {}
image_files = [ image_files = [
'sc-default.png', "sc-default.png",
'sc-talk.png', "sc-talk.png",
'sc-listen-1.png', "sc-listen-1.png",
'sc-think-1.png', "sc-think-1.png",
'sc-think-2.png', "sc-think-2.png",
'sc-think-3.png', "sc-think-3.png",
'sc-think-4.png' "sc-think-4.png",
] ]
script_dir = os.path.dirname(__file__) script_dir = os.path.dirname(__file__)
@@ -47,16 +52,17 @@ for file in image_files:
# When the bot isn't talking, show a static image of the cat listening # When the bot isn't talking, show a static image of the cat listening
quiet_frame = ImageFrame("", sprites["sc-listen-1.png"]) quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
# When the bot is talking, build an animation from two sprites # When the bot is talking, build an animation from two sprites
talking_list = [sprites['sc-default.png'], sprites['sc-talk.png']] talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
talking = [random.choice(talking_list) for x in range(30)] talking = [random.choice(talking_list) for x in range(30)]
talking_frame = SpriteFrame(images=talking) talking_frame = SpriteFrame(images=talking)
# TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing # TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing
thinking_list = [ thinking_list = [
sprites['sc-think-1.png'], sprites["sc-think-1.png"],
sprites['sc-think-2.png'], sprites["sc-think-2.png"],
sprites['sc-think-3.png'], sprites["sc-think-3.png"],
sprites['sc-think-4.png']] sprites["sc-think-4.png"],
]
thinking_frame = SpriteFrame(images=thinking_list) thinking_frame = SpriteFrame(images=thinking_list)
@@ -115,7 +121,7 @@ async def main(room_url: str, token):
mic_sample_rate=16000, mic_sample_rate=16000,
camera_enabled=True, camera_enabled=True,
camera_width=720, camera_width=720,
camera_height=1280 camera_height=1280,
) )
transport._mic_enabled = True transport._mic_enabled = True
transport._mic_sample_rate = 16000 transport._mic_sample_rate = 16000
@@ -126,25 +132,31 @@ async def main(room_url: str, token):
llm = AzureLLMService( llm = AzureLLMService(
api_key=os.getenv("AZURE_CHATGPT_API_KEY"), api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
model=os.getenv("AZURE_CHATGPT_MODEL")) model=os.getenv("AZURE_CHATGPT_MODEL"),
)
tts = ElevenLabsTTSService( tts = ElevenLabsTTSService(
aiohttp_session=session, aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"), api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="jBpfuIE2acCO8z3wKNLl") voice_id="jBpfuIE2acCO8z3wKNLl",
)
isa = ImageSyncAggregator() isa = ImageSyncAggregator()
@transport.event_handler("on_first_other_participant_joined") @transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport): async def on_first_other_participant_joined(transport):
await tts.say("Hi! If you want to talk to me, just say 'hey Santa Cat'.", transport.send_queue) await tts.say(
"Hi! If you want to talk to me, just say 'hey Santa Cat'.",
transport.send_queue,
)
async def handle_transcriptions(): async def handle_transcriptions():
messages = [ messages = [
{"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."}, {
"role": "system",
"content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long.",
},
] ]
tma_in = LLMUserContextAggregator( tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
messages, transport._my_participant_id
)
tma_out = LLMAssistantContextAggregator( tma_out = LLMAssistantContextAggregator(
messages, transport._my_participant_id messages, transport._my_participant_id
) )
@@ -155,16 +167,10 @@ async def main(room_url: str, token):
isa.run( isa.run(
tma_out.run( tma_out.run(
llm.run( llm.run(
tma_in.run( tma_in.run(ncf.run(tf.run(transport.get_receive_frames())))
ncf.run(
tf.run(
transport.get_receive_frames()
)
)
)
) )
) )
) ),
) )
async def starting_image(): async def starting_image():

View File

@@ -14,7 +14,7 @@ from typing import AsyncGenerator
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai") logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)

View File

@@ -1,10 +1,14 @@
import asyncio import asyncio
import logging
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.whisper_ai_services import WhisperSTTService from dailyai.services.whisper_ai_services import WhisperSTTService
from examples.support.runner import configure from examples.support.runner import configure
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str): async def main(room_url: str):
transport = DailyTransportService( transport = DailyTransportService(
@@ -14,7 +18,7 @@ async def main(room_url: str):
start_transcription=True, start_transcription=True,
mic_enabled=False, mic_enabled=False,
camera_enabled=False, camera_enabled=False,
speaker_enabled=True speaker_enabled=True,
) )
stt = WhisperSTTService() stt = WhisperSTTService()
@@ -28,9 +32,9 @@ async def main(room_url: str):
async def handle_speaker(): async def handle_speaker():
await stt.run_to_queue( await stt.run_to_queue(
transcription_output_queue, transcription_output_queue, transport.get_receive_frames()
transport.get_receive_frames()
) )
await asyncio.gather(transport.run(), handle_speaker(), handle_transcription()) await asyncio.gather(transport.run(), handle_speaker(), handle_transcription())

View File

@@ -1,11 +1,16 @@
import argparse import argparse
import asyncio import asyncio
import logging
import wave import wave
from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
from dailyai.services.local_transport_service import LocalTransportService from dailyai.services.local_transport_service import LocalTransportService
from dailyai.services.whisper_ai_services import WhisperSTTService from dailyai.services.whisper_ai_services import WhisperSTTService
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
async def main(room_url: str): async def main(room_url: str):
global transport global transport
@@ -17,7 +22,7 @@ async def main(room_url: str):
camera_enabled=False, camera_enabled=False,
speaker_enabled=True, speaker_enabled=True,
duration_minutes=meeting_duration_minutes, duration_minutes=meeting_duration_minutes,
start_transcription=True start_transcription=True,
) )
stt = WhisperSTTService() stt = WhisperSTTService()
transcription_output_queue = asyncio.Queue() transcription_output_queue = asyncio.Queue()

View File

@@ -2,35 +2,48 @@ import aiohttp
import asyncio import asyncio
import json import json
import random import random
import logging
import os import os
import re import re
import wave import wave
from typing import AsyncGenerator from typing import AsyncGenerator
from PIL import Image from PIL import Image
import sys
print('\n'.join(sys.path))
from dailyai.pipeline.pipeline import Pipeline from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.open_ai_services import OpenAILLMService from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator, UserResponseAggregator, LLMResponseAggregator from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMContextAggregator,
LLMUserContextAggregator,
UserResponseAggregator,
LLMResponseAggregator,
)
from examples.support.runner import configure from examples.support.runner import configure
from dailyai.pipeline.frames import LLMMessagesQueueFrame, TranscriptionQueueFrame, Frame, TextFrame, LLMFunctionCallFrame, LLMFunctionStartFrame, LLMResponseEndFrame, StartFrame, AudioFrame, SpriteFrame, ImageFrame from dailyai.pipeline.frames import (
LLMMessagesQueueFrame,
TranscriptionQueueFrame,
Frame,
TextFrame,
LLMFunctionCallFrame,
LLMFunctionStartFrame,
LLMResponseEndFrame,
StartFrame,
AudioFrame,
SpriteFrame,
ImageFrame,
)
from dailyai.services.ai_services import FrameLogger, AIService from dailyai.services.ai_services import FrameLogger, AIService
import logging logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logging.basicConfig(level=logging.INFO) logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
sounds = {} sounds = {}
sound_files = [ sound_files = ["clack-short.wav", "clack.wav", "clack-short-quiet.wav"]
'clack-short.wav',
'clack.wav',
'clack-short-quiet.wav'
]
script_dir = os.path.dirname(__file__) script_dir = os.path.dirname(__file__)
@@ -48,9 +61,11 @@ steps = [
{ {
"prompt": "Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function.", "prompt": "Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function.",
"run_async": False, "run_async": False,
"failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.", "tools": [{ "failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.",
"type": "function", "tools": [
"function": { {
"type": "function",
"function": {
"name": "verify_birthday", "name": "verify_birthday",
"description": "Use this function to verify the user has provided their correct birthday.", "description": "Use this function to verify the user has provided their correct birthday.",
"parameters": { "parameters": {
@@ -58,18 +73,21 @@ steps = [
"properties": { "properties": {
"birthday": { "birthday": {
"type": "string", "type": "string",
"description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function." "description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function.",
} }
} },
} },
},
} }
}]}, ],
},
{ {
"prompt": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages.", "prompt": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages.",
"run_async": True, "run_async": True,
"tools": [{ "tools": [
"type": "function", {
"function": { "type": "function",
"function": {
"name": "list_prescriptions", "name": "list_prescriptions",
"description": "Once the user has provided a list of their prescription medications, call this function.", "description": "Once the user has provided a list of their prescription medications, call this function.",
"parameters": { "parameters": {
@@ -82,19 +100,20 @@ steps = [
"properties": { "properties": {
"medication": { "medication": {
"type": "string", "type": "string",
"description": "The medication's name" "description": "The medication's name",
}, },
"dosage": { "dosage": {
"type": "string", "type": "string",
"description": "The prescription's dosage" "description": "The prescription's dosage",
} },
} },
} },
} }
} },
} },
},
} }
}] ],
}, },
{ {
"prompt": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function.", "prompt": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function.",
@@ -115,16 +134,16 @@ steps = [
"properties": { "properties": {
"name": { "name": {
"type": "string", "type": "string",
"description": "What the user is allergic to" "description": "What the user is allergic to",
} }
} },
} },
} }
} },
} },
} },
} }
] ],
}, },
{ {
"prompt": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function.", "prompt": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function.",
@@ -145,14 +164,14 @@ steps = [
"properties": { "properties": {
"name": { "name": {
"type": "string", "type": "string",
"description": "The user's medical condition" "description": "The user's medical condition",
} }
} },
} },
} }
} },
} },
} },
}, },
], ],
}, },
@@ -175,20 +194,23 @@ steps = [
"properties": { "properties": {
"name": { "name": {
"type": "string", "type": "string",
"description": "The user's reason for visiting the doctor" "description": "The user's reason for visiting the doctor",
} }
} },
} },
} }
} },
} },
} },
} }
] ],
}, },
{"prompt": "Now, thank the user and end the conversation.", {
"run_async": True, "tools": []}, "prompt": "Now, thank the user and end the conversation.",
{"prompt": "", "run_async": True, "tools": []} "run_async": True,
"tools": [],
},
{"prompt": "", "run_async": True, "tools": []},
] ]
current_step = 0 current_step = 0
@@ -219,15 +241,15 @@ class ChecklistProcessor(AIService):
"list_prescriptions", "list_prescriptions",
"list_allergies", "list_allergies",
"list_conditions", "list_conditions",
"list_visit_reasons" "list_visit_reasons",
] ]
messages.append( messages.append(
{"role": "system", "content": f"{self._id} {steps[0]['prompt']}"}) {"role": "system", "content": f"{self._id} {steps[0]['prompt']}"}
)
def verify_birthday(self, args): def verify_birthday(self, args):
return args['birthday'] == "1983-01-01" return args["birthday"] == "1983-01-01"
def list_prescriptions(self, args): def list_prescriptions(self, args):
# print(f"--- Prescriptions: {args['prescriptions']}\n") # print(f"--- Prescriptions: {args['prescriptions']}\n")
@@ -250,18 +272,21 @@ class ChecklistProcessor(AIService):
this_step = steps[current_step] this_step = steps[current_step]
# TODO-CB: forcing a global here :/ # TODO-CB: forcing a global here :/
self._tools.clear() self._tools.clear()
self._tools.extend(this_step['tools']) self._tools.extend(this_step["tools"])
if isinstance(frame, LLMFunctionStartFrame): if isinstance(frame, LLMFunctionStartFrame):
print(f"... Preparing function call: {frame.function_name}") print(f"... Preparing function call: {frame.function_name}")
self._function_name = frame.function_name self._function_name = frame.function_name
if this_step['run_async']: if this_step["run_async"]:
# Get the LLM talking about the next step before getting the rest # Get the LLM talking about the next step before getting the rest
# of the function call completion # of the function call completion
current_step += 1 current_step += 1
self._messages.append({ self._messages.append(
"role": "system", "content": steps[current_step]['prompt']}) {"role": "system", "content": steps[current_step]["prompt"]}
)
yield LLMMessagesQueueFrame(self._messages) yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"): async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame yield frame
else: else:
# Insert a quick response while we run the function # Insert a quick response while we run the function
@@ -270,29 +295,37 @@ class ChecklistProcessor(AIService):
elif isinstance(frame, LLMFunctionCallFrame): elif isinstance(frame, LLMFunctionCallFrame):
if frame.function_name and frame.arguments: if frame.function_name and frame.arguments:
print( print(f"--> Calling function: {frame.function_name} with arguments:")
f"--> Calling function: {frame.function_name} with arguments:") pretty_json = re.sub(
pretty_json = re.sub("\n", "\n ", json.dumps( "\n", "\n ", json.dumps(json.loads(frame.arguments), indent=2)
json.loads(frame.arguments), indent=2)) )
print(f"--> {pretty_json}\n") print(f"--> {pretty_json}\n")
if not frame.function_name in self._functions: if not frame.function_name in self._functions:
raise Exception(f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions.") raise Exception(
f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions."
)
fn = getattr(self, frame.function_name) fn = getattr(self, frame.function_name)
result = fn(json.loads(frame.arguments)) result = fn(json.loads(frame.arguments))
if not this_step['run_async']: if not this_step["run_async"]:
if result: if result:
current_step += 1 current_step += 1
self._messages.append({ self._messages.append(
"role": "system", "content": steps[current_step]['prompt']}) {"role": "system", "content": steps[current_step]["prompt"]}
)
yield LLMMessagesQueueFrame(self._messages) yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"): async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame yield frame
else: else:
self._messages.append({ self._messages.append(
"role": "system", "content": this_step['failed']}) {"role": "system", "content": this_step["failed"]}
)
yield LLMMessagesQueueFrame(self._messages) yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"): async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame yield frame
print(f"<-- Verify result: {result}\n") print(f"<-- Verify result: {result}\n")
@@ -315,7 +348,7 @@ async def main(room_url: str, token):
mic_sample_rate=16000, mic_sample_rate=16000,
camera_enabled=False, camera_enabled=False,
start_transcription=True, start_transcription=True,
vad_enabled=True vad_enabled=True,
) )
# TODO-CB: Go back to vad_enabled # TODO-CB: Go back to vad_enabled
@@ -324,12 +357,18 @@ async def main(room_url: str, token):
# llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv( # llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv(
# "AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) # "AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
llm = OpenAILLMService(api_key=os.getenv( llm = OpenAILLMService(
"OPENAI_CHATGPT_API_KEY"), model="gpt-4-1106-preview", tools=tools) # gpt-4-1106-preview api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
model="gpt-4-1106-preview",
tools=tools,
) # gpt-4-1106-preview
# tts = AzureTTSService(api_key=os.getenv( # tts = AzureTTSService(api_key=os.getenv(
# "AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) # "AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv( tts = ElevenLabsTTSService(
"ELEVENLABS_API_KEY"), voice_id="XrExE9yKIg1WjnnlVkGX") # matilda aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="XrExE9yKIg1WjnnlVkGX",
) # matilda
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv( # tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv(
# "DEEPGRAM_API_KEY"), voice="aura-asteria-en") # "DEEPGRAM_API_KEY"), voice="aura-asteria-en")
@@ -345,34 +384,23 @@ async def main(room_url: str, token):
# TODO-CB: Make sure this message gets into the context somehow # TODO-CB: Make sure this message gets into the context somehow
await tts.run_to_queue( await tts.run_to_queue(
transport.send_queue, transport.send_queue,
llm.run([LLMMessagesQueueFrame(messages)]), llm.run([LLMMessagesQueueFrame(messages)]),
) )
async def handle_intake(): async def handle_intake():
pipeline = Pipeline( pipeline = Pipeline(processors=[fl, llm, fl2, checklist, tts])
processors=[ await transport.run_interruptible_pipeline(
fl, pipeline,
llm, post_processor=LLMResponseAggregator(messages),
fl2, pre_processor=UserResponseAggregator(messages),
checklist,
tts
]
) )
await transport.run_interruptible_pipeline(pipeline,
post_processor=LLMResponseAggregator(
messages
),
pre_processor=UserResponseAggregator(messages)
)
transport.transcription_settings["extra"]["endpointing"] = True transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True transport.transcription_settings["extra"]["punctuate"] = True
try: try:
await asyncio.gather(transport.run(), handle_intake()) await asyncio.gather(transport.run(), handle_intake())
except (asyncio.CancelledError, KeyboardInterrupt): except (asyncio.CancelledError, KeyboardInterrupt):
print('whoops') print("whoops")
transport.stop() transport.stop()