cleaned up example logging (#46)
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
@@ -8,6 +9,9 @@ from dailyai.services.playht_ai_service import PlayHTAIService
|
|||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
@@ -21,24 +25,14 @@ async def main(room_url):
|
|||||||
# to the daily-python basic API
|
# to the daily-python basic API
|
||||||
meeting_duration_minutes = 5
|
meeting_duration_minutes = 5
|
||||||
transport = DailyTransportService(
|
transport = DailyTransportService(
|
||||||
room_url,
|
room_url, None, "Say One Thing", meeting_duration_minutes, mic_enabled=True
|
||||||
None,
|
|
||||||
"Say One Thing",
|
|
||||||
meeting_duration_minutes,
|
|
||||||
mic_enabled=True
|
|
||||||
)
|
)
|
||||||
|
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
|
||||||
"""
|
|
||||||
tts = PlayHTAIService(
|
|
||||||
api_key=os.getenv("PLAY_HT_API_KEY"),
|
|
||||||
user_id=os.getenv("PLAY_HT_USER_ID"),
|
|
||||||
voice_url=os.getenv("PLAY_HT_VOICE_URL"),
|
|
||||||
)
|
)
|
||||||
"""
|
|
||||||
|
|
||||||
# Register an event handler so we can play the audio when the participant joins.
|
# Register an event handler so we can play the audio when the participant joins.
|
||||||
@transport.event_handler("on_participant_joined")
|
@transport.event_handler("on_participant_joined")
|
||||||
@@ -56,7 +50,7 @@ async def main(room_url):
|
|||||||
await transport.stop_when_done()
|
await transport.stop_when_done()
|
||||||
|
|
||||||
await transport.run()
|
await transport.run()
|
||||||
del(tts)
|
del tts
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -1,17 +1,20 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
from dailyai.services.local_transport_service import LocalTransportService
|
from dailyai.services.local_transport_service import LocalTransportService
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
async def main():
|
async def main():
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
meeting_duration_minutes = 1
|
meeting_duration_minutes = 1
|
||||||
transport = LocalTransportService(
|
transport = LocalTransportService(
|
||||||
duration_minutes=meeting_duration_minutes,
|
duration_minutes=meeting_duration_minutes, mic_enabled=True
|
||||||
mic_enabled=True
|
|
||||||
)
|
)
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
|
import logging
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
|
||||||
@@ -11,6 +12,9 @@ from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
|||||||
from dailyai.services.open_ai_services import OpenAILLMService
|
from dailyai.services.open_ai_services import OpenAILLMService
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
@@ -20,25 +24,23 @@ async def main(room_url):
|
|||||||
None,
|
None,
|
||||||
"Say One Thing From an LLM",
|
"Say One Thing From an LLM",
|
||||||
duration_minutes=meeting_duration_minutes,
|
duration_minutes=meeting_duration_minutes,
|
||||||
mic_enabled=True
|
mic_enabled=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
|
||||||
# tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
|
)
|
||||||
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE"))
|
llm = OpenAILLMService(
|
||||||
|
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
|
||||||
llm = AzureLLMService(
|
)
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
messages = [
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
{
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
"role": "system",
|
||||||
# llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY"))
|
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world.",
|
||||||
messages = [{
|
}
|
||||||
"role": "system",
|
]
|
||||||
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world."
|
|
||||||
}]
|
|
||||||
tts_task = asyncio.create_task(
|
tts_task = asyncio.create_task(
|
||||||
tts.run_to_queue(
|
tts.run_to_queue(
|
||||||
transport.send_queue,
|
transport.send_queue,
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from dailyai.pipeline.frames import TextFrame
|
from dailyai.pipeline.frames import TextFrame
|
||||||
@@ -10,6 +11,9 @@ from dailyai.services.azure_ai_services import AzureImageGenServiceREST
|
|||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
local_joined = False
|
local_joined = False
|
||||||
participant_joined = False
|
participant_joined = False
|
||||||
|
|
||||||
@@ -25,21 +29,23 @@ async def main(room_url):
|
|||||||
mic_enabled=False,
|
mic_enabled=False,
|
||||||
camera_enabled=True,
|
camera_enabled=True,
|
||||||
camera_width=1024,
|
camera_width=1024,
|
||||||
camera_height=1024
|
camera_height=1024,
|
||||||
)
|
)
|
||||||
|
|
||||||
imagegen = FalImageGenService(
|
imagegen = FalImageGenService(
|
||||||
image_size="1024x1024",
|
image_size="1024x1024",
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
key_id=os.getenv("FAL_KEY_ID"),
|
key_id=os.getenv("FAL_KEY_ID"),
|
||||||
key_secret=os.getenv("FAL_KEY_SECRET"))
|
key_secret=os.getenv("FAL_KEY_SECRET"),
|
||||||
|
)
|
||||||
# imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
|
# imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024")
|
||||||
# imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
|
# imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL"))
|
||||||
|
|
||||||
image_task = asyncio.create_task(
|
image_task = asyncio.create_task(
|
||||||
imagegen.run_to_queue(
|
imagegen.run_to_queue(
|
||||||
transport.send_queue, [
|
transport.send_queue, [TextFrame("a cat in the style of picasso")]
|
||||||
TextFrame("a cat in the style of picasso")]))
|
)
|
||||||
|
)
|
||||||
|
|
||||||
@transport.event_handler("on_first_other_participant_joined")
|
@transport.event_handler("on_first_other_participant_joined")
|
||||||
async def on_first_other_participant_joined(transport):
|
async def on_first_other_participant_joined(transport):
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import tkinter as tk
|
import tkinter as tk
|
||||||
@@ -8,6 +9,10 @@ from dailyai.pipeline.frames import TextFrame
|
|||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
from dailyai.services.local_transport_service import LocalTransportService
|
from dailyai.services.local_transport_service import LocalTransportService
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
local_joined = False
|
local_joined = False
|
||||||
participant_joined = False
|
participant_joined = False
|
||||||
|
|
||||||
@@ -46,5 +51,6 @@ async def main():
|
|||||||
|
|
||||||
await asyncio.gather(transport.run(), image_task, run_tk())
|
await asyncio.gather(transport.run(), image_task, run_tk())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
asyncio.run(main())
|
asyncio.run(main())
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
import aiohttp
|
import aiohttp
|
||||||
@@ -8,9 +9,12 @@ from dailyai.services.daily_transport_service import DailyTransportService
|
|||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
|
from dailyai.pipeline.frames import EndFrame, LLMMessagesQueueFrame
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url: str):
|
async def main(room_url: str):
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
@@ -21,20 +25,23 @@ async def main(room_url: str):
|
|||||||
duration_minutes=1,
|
duration_minutes=1,
|
||||||
mic_enabled=True,
|
mic_enabled=True,
|
||||||
mic_sample_rate=16000,
|
mic_sample_rate=16000,
|
||||||
camera_enabled=False
|
camera_enabled=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
llm = AzureLLMService(
|
llm = AzureLLMService(
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
model=os.getenv("AZURE_CHATGPT_MODEL"),
|
||||||
|
)
|
||||||
azure_tts = AzureTTSService(
|
azure_tts = AzureTTSService(
|
||||||
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
||||||
region=os.getenv("AZURE_SPEECH_REGION"))
|
region=os.getenv("AZURE_SPEECH_REGION"),
|
||||||
|
)
|
||||||
elevenlabs_tts = ElevenLabsTTSService(
|
elevenlabs_tts = ElevenLabsTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
voice_id=os.getenv("ELEVENLABS_VOICE_ID"))
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
|
||||||
|
)
|
||||||
|
|
||||||
messages = [{"role": "system", "content": "tell the user a joke about llamas"}]
|
messages = [{"role": "system", "content": "tell the user a joke about llamas"}]
|
||||||
|
|
||||||
@@ -43,14 +50,19 @@ async def main(room_url: str):
|
|||||||
# speak the LLM response.
|
# speak the LLM response.
|
||||||
buffer_queue = asyncio.Queue()
|
buffer_queue = asyncio.Queue()
|
||||||
source_queue = asyncio.Queue()
|
source_queue = asyncio.Queue()
|
||||||
pipeline = Pipeline(source = source_queue, sink=buffer_queue, processors=[llm, elevenlabs_tts])
|
pipeline = Pipeline(
|
||||||
|
source=source_queue, sink=buffer_queue, processors=[llm, elevenlabs_tts]
|
||||||
|
)
|
||||||
await source_queue.put(LLMMessagesQueueFrame(messages))
|
await source_queue.put(LLMMessagesQueueFrame(messages))
|
||||||
await source_queue.put(EndFrame())
|
await source_queue.put(EndFrame())
|
||||||
pipeline_run_task = pipeline.run_pipeline()
|
pipeline_run_task = pipeline.run_pipeline()
|
||||||
|
|
||||||
@transport.event_handler("on_first_other_participant_joined")
|
@transport.event_handler("on_first_other_participant_joined")
|
||||||
async def on_first_other_participant_joined(transport):
|
async def on_first_other_participant_joined(transport):
|
||||||
await azure_tts.say("My friend the LLM is now going to tell a joke about llamas.", transport.send_queue)
|
await azure_tts.say(
|
||||||
|
"My friend the LLM is now going to tell a joke about llamas.",
|
||||||
|
transport.send_queue,
|
||||||
|
)
|
||||||
|
|
||||||
async def buffer_to_send_queue():
|
async def buffer_to_send_queue():
|
||||||
while True:
|
while True:
|
||||||
|
|||||||
@@ -2,11 +2,27 @@ import asyncio
|
|||||||
from re import S
|
from re import S
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import os
|
import os
|
||||||
from dailyai.pipeline.aggregators import GatedAggregator, LLMFullResponseAggregator, ParallelPipeline, SentenceAggregator
|
import logging
|
||||||
|
|
||||||
from dailyai.pipeline.frames import AudioFrame, EndFrame, ImageFrame, LLMMessagesQueueFrame, LLMResponseStartFrame
|
from dailyai.pipeline.aggregators import (
|
||||||
|
GatedAggregator,
|
||||||
|
LLMFullResponseAggregator,
|
||||||
|
ParallelPipeline,
|
||||||
|
SentenceAggregator,
|
||||||
|
)
|
||||||
|
from dailyai.pipeline.frames import (
|
||||||
|
AudioFrame,
|
||||||
|
EndFrame,
|
||||||
|
ImageFrame,
|
||||||
|
LLMMessagesQueueFrame,
|
||||||
|
LLMResponseStartFrame,
|
||||||
|
)
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureImageGenServiceREST, AzureTTSService
|
from dailyai.services.azure_ai_services import (
|
||||||
|
AzureLLMService,
|
||||||
|
AzureImageGenServiceREST,
|
||||||
|
AzureTTSService,
|
||||||
|
)
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
@@ -14,6 +30,10 @@ from dailyai.services.open_ai_services import OpenAIImageGenService
|
|||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
@@ -27,23 +47,26 @@ async def main(room_url):
|
|||||||
camera_enabled=True,
|
camera_enabled=True,
|
||||||
mic_sample_rate=16000,
|
mic_sample_rate=16000,
|
||||||
camera_width=1024,
|
camera_width=1024,
|
||||||
camera_height=1024
|
camera_height=1024,
|
||||||
)
|
)
|
||||||
|
|
||||||
llm = AzureLLMService(
|
llm = AzureLLMService(
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
model=os.getenv("AZURE_CHATGPT_MODEL"),
|
||||||
|
)
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
voice_id="ErXwobaYiN019PkySvjV")
|
voice_id="ErXwobaYiN019PkySvjV",
|
||||||
|
)
|
||||||
|
|
||||||
dalle = FalImageGenService(
|
dalle = FalImageGenService(
|
||||||
image_size="1024x1024",
|
image_size="1024x1024",
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
key_id=os.getenv("FAL_KEY_ID"),
|
key_id=os.getenv("FAL_KEY_ID"),
|
||||||
key_secret=os.getenv("FAL_KEY_SECRET"))
|
key_secret=os.getenv("FAL_KEY_SECRET"),
|
||||||
|
)
|
||||||
|
|
||||||
source_queue = asyncio.Queue()
|
source_queue = asyncio.Queue()
|
||||||
|
|
||||||
@@ -101,6 +124,7 @@ async def main(room_url):
|
|||||||
|
|
||||||
await transport.run()
|
await transport.run()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
(url, token) = configure()
|
(url, token) = configure()
|
||||||
asyncio.run(main(url))
|
asyncio.run(main(url))
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import aiohttp
|
import aiohttp
|
||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import tkinter as tk
|
import tkinter as tk
|
||||||
import os
|
import os
|
||||||
|
|
||||||
@@ -10,6 +11,10 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
|||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
from dailyai.services.local_transport_service import LocalTransportService
|
from dailyai.services.local_transport_service import LocalTransportService
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
@@ -67,9 +72,7 @@ async def main(room_url):
|
|||||||
to_speak = f"{month}: {image_description}"
|
to_speak = f"{month}: {image_description}"
|
||||||
audio_task = asyncio.create_task(get_all_audio(to_speak))
|
audio_task = asyncio.create_task(get_all_audio(to_speak))
|
||||||
image_task = asyncio.create_task(dalle.run_image_gen(image_description))
|
image_task = asyncio.create_task(dalle.run_image_gen(image_description))
|
||||||
(audio, image_data) = await asyncio.gather(
|
(audio, image_data) = await asyncio.gather(audio_task, image_task)
|
||||||
audio_task, image_task
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"month": month,
|
"month": month,
|
||||||
@@ -123,6 +126,7 @@ async def main(room_url):
|
|||||||
|
|
||||||
await asyncio.gather(transport.run(), show_images(), run_tk())
|
await asyncio.gather(transport.run(), show_images(), run_tk())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
|
|||||||
@@ -1,65 +1,81 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
import aiohttp
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
|
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
from dailyai.services.open_ai_services import OpenAILLMService
|
||||||
from dailyai.services.ai_services import FrameLogger
|
from dailyai.services.ai_services import FrameLogger
|
||||||
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator
|
from dailyai.pipeline.aggregators import (
|
||||||
|
LLMAssistantContextAggregator,
|
||||||
|
LLMUserContextAggregator,
|
||||||
|
)
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url: str, token):
|
async def main(room_url: str, token):
|
||||||
transport = DailyTransportService(
|
async with aiohttp.ClientSession() as session:
|
||||||
room_url,
|
transport = DailyTransportService(
|
||||||
token,
|
room_url,
|
||||||
"Respond bot",
|
token,
|
||||||
duration_minutes=5,
|
"Respond bot",
|
||||||
start_transcription=True,
|
duration_minutes=5,
|
||||||
mic_enabled=True,
|
start_transcription=True,
|
||||||
mic_sample_rate=16000,
|
mic_enabled=True,
|
||||||
camera_enabled=False,
|
mic_sample_rate=16000,
|
||||||
vad_enabled=True
|
camera_enabled=False,
|
||||||
)
|
vad_enabled=True,
|
||||||
|
|
||||||
llm = AzureLLMService(
|
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
|
||||||
tts = AzureTTSService(
|
|
||||||
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
|
||||||
region=os.getenv("AZURE_SPEECH_REGION"))
|
|
||||||
fl = FrameLogger("Inner")
|
|
||||||
fl2 = FrameLogger("Outer")
|
|
||||||
@transport.event_handler("on_first_other_participant_joined")
|
|
||||||
async def on_first_other_participant_joined(transport):
|
|
||||||
await tts.say("Hi, I'm listening!", transport.send_queue)
|
|
||||||
|
|
||||||
async def handle_transcriptions():
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "system",
|
|
||||||
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
|
|
||||||
},
|
|
||||||
]
|
|
||||||
|
|
||||||
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
|
|
||||||
tma_out = LLMAssistantContextAggregator(messages, transport._my_participant_id)
|
|
||||||
pipeline = Pipeline(
|
|
||||||
processors=[
|
|
||||||
fl,
|
|
||||||
tma_in,
|
|
||||||
llm,
|
|
||||||
fl2,
|
|
||||||
tts,
|
|
||||||
tma_out,
|
|
||||||
],
|
|
||||||
)
|
)
|
||||||
await transport.run_uninterruptible_pipeline(pipeline)
|
|
||||||
|
|
||||||
transport.transcription_settings["extra"]["endpointing"] = True
|
tts = ElevenLabsTTSService(
|
||||||
transport.transcription_settings["extra"]["punctuate"] = True
|
aiohttp_session=session,
|
||||||
await asyncio.gather(transport.run(), handle_transcriptions())
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
|
voice_id=os.getenv("ELEVENLABS_VOICE_ID"),
|
||||||
|
)
|
||||||
|
|
||||||
|
llm = OpenAILLMService(
|
||||||
|
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"), model="gpt-4-turbo-preview"
|
||||||
|
)
|
||||||
|
fl = FrameLogger("Inner")
|
||||||
|
fl2 = FrameLogger("Outer")
|
||||||
|
|
||||||
|
@transport.event_handler("on_first_other_participant_joined")
|
||||||
|
async def on_first_other_participant_joined(transport):
|
||||||
|
await tts.say("Hi, I'm listening!", transport.send_queue)
|
||||||
|
|
||||||
|
async def handle_transcriptions():
|
||||||
|
messages = [
|
||||||
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
|
||||||
|
tma_out = LLMAssistantContextAggregator(
|
||||||
|
messages, transport._my_participant_id
|
||||||
|
)
|
||||||
|
pipeline = Pipeline(
|
||||||
|
processors=[
|
||||||
|
fl,
|
||||||
|
tma_in,
|
||||||
|
llm,
|
||||||
|
fl2,
|
||||||
|
tts,
|
||||||
|
tma_out,
|
||||||
|
],
|
||||||
|
)
|
||||||
|
await transport.run_uninterruptible_pipeline(pipeline)
|
||||||
|
|
||||||
|
transport.transcription_settings["extra"]["endpointing"] = True
|
||||||
|
transport.transcription_settings["extra"]["punctuate"] = True
|
||||||
|
await asyncio.gather(transport.run(), handle_transcriptions())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -1,23 +1,29 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
import os
|
import os
|
||||||
|
import logging
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
import aiohttp
|
import aiohttp
|
||||||
import requests
|
import requests
|
||||||
import time
|
import time
|
||||||
import urllib.parse
|
import urllib.parse
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from dailyai.pipeline.frames import ImageFrame, Frame
|
|
||||||
|
|
||||||
|
from dailyai.pipeline.frames import ImageFrame, Frame
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
from dailyai.services.ai_services import AIService
|
from dailyai.services.ai_services import AIService
|
||||||
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMUserContextAggregator
|
from dailyai.pipeline.aggregators import (
|
||||||
|
LLMAssistantContextAggregator,
|
||||||
|
LLMUserContextAggregator,
|
||||||
|
)
|
||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
class ImageSyncAggregator(AIService):
|
class ImageSyncAggregator(AIService):
|
||||||
def __init__(self, speaking_path: str, waiting_path: str):
|
def __init__(self, speaking_path: str, waiting_path: str):
|
||||||
@@ -50,15 +56,18 @@ async def main(room_url: str, token):
|
|||||||
llm = AzureLLMService(
|
llm = AzureLLMService(
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
model=os.getenv("AZURE_CHATGPT_MODEL"),
|
||||||
|
)
|
||||||
tts = AzureTTSService(
|
tts = AzureTTSService(
|
||||||
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
||||||
region=os.getenv("AZURE_SPEECH_REGION"))
|
region=os.getenv("AZURE_SPEECH_REGION"),
|
||||||
|
)
|
||||||
img = FalImageGenService(
|
img = FalImageGenService(
|
||||||
image_size="1024x1024",
|
image_size="1024x1024",
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
key_id=os.getenv("FAL_KEY_ID"),
|
key_id=os.getenv("FAL_KEY_ID"),
|
||||||
key_secret=os.getenv("FAL_KEY_SECRET"))
|
key_secret=os.getenv("FAL_KEY_SECRET"),
|
||||||
|
)
|
||||||
|
|
||||||
async def get_images():
|
async def get_images():
|
||||||
get_speaking_task = asyncio.create_task(
|
get_speaking_task = asyncio.create_task(
|
||||||
@@ -80,12 +89,13 @@ async def main(room_url: str, token):
|
|||||||
|
|
||||||
async def handle_transcriptions():
|
async def handle_transcriptions():
|
||||||
messages = [
|
messages = [
|
||||||
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
tma_in = LLMUserContextAggregator(
|
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
|
||||||
messages, transport._my_participant_id
|
|
||||||
)
|
|
||||||
tma_out = LLMAssistantContextAggregator(
|
tma_out = LLMAssistantContextAggregator(
|
||||||
messages, transport._my_participant_id
|
messages, transport._my_participant_id
|
||||||
)
|
)
|
||||||
@@ -96,14 +106,8 @@ async def main(room_url: str, token):
|
|||||||
await tts.run_to_queue(
|
await tts.run_to_queue(
|
||||||
transport.send_queue,
|
transport.send_queue,
|
||||||
image_sync_aggregator.run(
|
image_sync_aggregator.run(
|
||||||
tma_out.run(
|
tma_out.run(llm.run(tma_in.run(transport.get_receive_frames())))
|
||||||
llm.run(
|
),
|
||||||
tma_in.run(
|
|
||||||
transport.get_receive_frames()
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
transport.transcription_settings["extra"]["punctuate"] = True
|
transport.transcription_settings["extra"]["punctuate"] = True
|
||||||
|
|||||||
@@ -1,14 +1,23 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
import aiohttp
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMResponseAggregator, LLMUserContextAggregator, UserResponseAggregator
|
from dailyai.pipeline.aggregators import (
|
||||||
|
LLMAssistantContextAggregator,
|
||||||
|
LLMResponseAggregator,
|
||||||
|
LLMUserContextAggregator,
|
||||||
|
UserResponseAggregator,
|
||||||
|
)
|
||||||
|
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
from dailyai.services.ai_services import FrameLogger
|
from dailyai.services.ai_services import FrameLogger
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
|
from examples.support.runner import configure
|
||||||
|
|
||||||
from support.runner import configure
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url: str, token):
|
async def main(room_url: str, token):
|
||||||
@@ -28,10 +37,12 @@ async def main(room_url: str, token):
|
|||||||
llm = AzureLLMService(
|
llm = AzureLLMService(
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
model=os.getenv("AZURE_CHATGPT_MODEL"),
|
||||||
|
)
|
||||||
tts = AzureTTSService(
|
tts = AzureTTSService(
|
||||||
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
||||||
region=os.getenv("AZURE_SPEECH_REGION"))
|
region=os.getenv("AZURE_SPEECH_REGION"),
|
||||||
|
)
|
||||||
|
|
||||||
pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
|
pipeline = Pipeline([FrameLogger(), llm, FrameLogger(), tts])
|
||||||
|
|
||||||
@@ -41,17 +52,16 @@ async def main(room_url: str, token):
|
|||||||
|
|
||||||
async def run_conversation():
|
async def run_conversation():
|
||||||
messages = [
|
messages = [
|
||||||
{"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."},
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way.",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
await transport.run_interruptible_pipeline(
|
await transport.run_interruptible_pipeline(
|
||||||
pipeline,
|
pipeline,
|
||||||
post_processor=LLMResponseAggregator(
|
post_processor=LLMResponseAggregator(messages),
|
||||||
messages
|
pre_processor=UserResponseAggregator(messages),
|
||||||
),
|
|
||||||
pre_processor=UserResponseAggregator(
|
|
||||||
messages
|
|
||||||
),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
transport.transcription_settings["extra"]["punctuate"] = False
|
transport.transcription_settings["extra"]["punctuate"] = False
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import aiohttp
|
import aiohttp
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
@@ -7,9 +8,12 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
|||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
from dailyai.pipeline.frames import AudioFrame, ImageFrame
|
from dailyai.pipeline.frames import AudioFrame, ImageFrame
|
||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url: str):
|
async def main(room_url: str):
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
@@ -22,33 +26,41 @@ async def main(room_url: str):
|
|||||||
mic_sample_rate=16000,
|
mic_sample_rate=16000,
|
||||||
camera_enabled=True,
|
camera_enabled=True,
|
||||||
camera_width=1024,
|
camera_width=1024,
|
||||||
camera_height=1024
|
camera_height=1024,
|
||||||
)
|
)
|
||||||
|
|
||||||
llm = AzureLLMService(
|
llm = AzureLLMService(
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
model=os.getenv("AZURE_CHATGPT_MODEL"),
|
||||||
|
)
|
||||||
tts1 = AzureTTSService(
|
tts1 = AzureTTSService(
|
||||||
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
api_key=os.getenv("AZURE_SPEECH_API_KEY"),
|
||||||
region=os.getenv("AZURE_SPEECH_REGION"))
|
region=os.getenv("AZURE_SPEECH_REGION"),
|
||||||
|
)
|
||||||
tts2 = ElevenLabsTTSService(
|
tts2 = ElevenLabsTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
voice_id="jBpfuIE2acCO8z3wKNLl")
|
voice_id="jBpfuIE2acCO8z3wKNLl",
|
||||||
|
)
|
||||||
dalle = FalImageGenService(
|
dalle = FalImageGenService(
|
||||||
image_size="1024x1024",
|
image_size="1024x1024",
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
key_id=os.getenv("FAL_KEY_ID"),
|
key_id=os.getenv("FAL_KEY_ID"),
|
||||||
key_secret=os.getenv("FAL_KEY_SECRET"))
|
key_secret=os.getenv("FAL_KEY_SECRET"),
|
||||||
|
)
|
||||||
|
|
||||||
bot1_messages = [
|
bot1_messages = [
|
||||||
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."},
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long.",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
bot2_messages = [
|
bot2_messages = [
|
||||||
{
|
{
|
||||||
"role": "system",
|
"role": "system",
|
||||||
"content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."},
|
"content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich.",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
async def get_bot1_statement():
|
async def get_bot1_statement():
|
||||||
|
|||||||
@@ -1,15 +1,18 @@
|
|||||||
import aiohttp
|
import aiohttp
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import random
|
import random
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.azure_ai_services import AzureLLMService
|
from dailyai.services.azure_ai_services import AzureLLMService
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
from dailyai.pipeline.aggregators import LLMUserContextAggregator, LLMAssistantContextAggregator
|
from dailyai.pipeline.aggregators import (
|
||||||
|
LLMUserContextAggregator,
|
||||||
|
LLMAssistantContextAggregator,
|
||||||
|
)
|
||||||
from dailyai.pipeline.frames import (
|
from dailyai.pipeline.frames import (
|
||||||
Frame,
|
Frame,
|
||||||
TextFrame,
|
TextFrame,
|
||||||
@@ -18,19 +21,21 @@ from dailyai.pipeline.frames import (
|
|||||||
TranscriptionQueueFrame,
|
TranscriptionQueueFrame,
|
||||||
)
|
)
|
||||||
from dailyai.services.ai_services import AIService
|
from dailyai.services.ai_services import AIService
|
||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
sprites = {}
|
sprites = {}
|
||||||
image_files = [
|
image_files = [
|
||||||
'sc-default.png',
|
"sc-default.png",
|
||||||
'sc-talk.png',
|
"sc-talk.png",
|
||||||
'sc-listen-1.png',
|
"sc-listen-1.png",
|
||||||
'sc-think-1.png',
|
"sc-think-1.png",
|
||||||
'sc-think-2.png',
|
"sc-think-2.png",
|
||||||
'sc-think-3.png',
|
"sc-think-3.png",
|
||||||
'sc-think-4.png'
|
"sc-think-4.png",
|
||||||
]
|
]
|
||||||
|
|
||||||
script_dir = os.path.dirname(__file__)
|
script_dir = os.path.dirname(__file__)
|
||||||
@@ -47,16 +52,17 @@ for file in image_files:
|
|||||||
# When the bot isn't talking, show a static image of the cat listening
|
# When the bot isn't talking, show a static image of the cat listening
|
||||||
quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
|
quiet_frame = ImageFrame("", sprites["sc-listen-1.png"])
|
||||||
# When the bot is talking, build an animation from two sprites
|
# When the bot is talking, build an animation from two sprites
|
||||||
talking_list = [sprites['sc-default.png'], sprites['sc-talk.png']]
|
talking_list = [sprites["sc-default.png"], sprites["sc-talk.png"]]
|
||||||
talking = [random.choice(talking_list) for x in range(30)]
|
talking = [random.choice(talking_list) for x in range(30)]
|
||||||
talking_frame = SpriteFrame(images=talking)
|
talking_frame = SpriteFrame(images=talking)
|
||||||
|
|
||||||
# TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing
|
# TODO: Support "thinking" as soon as we get a valid transcript, while LLM is processing
|
||||||
thinking_list = [
|
thinking_list = [
|
||||||
sprites['sc-think-1.png'],
|
sprites["sc-think-1.png"],
|
||||||
sprites['sc-think-2.png'],
|
sprites["sc-think-2.png"],
|
||||||
sprites['sc-think-3.png'],
|
sprites["sc-think-3.png"],
|
||||||
sprites['sc-think-4.png']]
|
sprites["sc-think-4.png"],
|
||||||
|
]
|
||||||
thinking_frame = SpriteFrame(images=thinking_list)
|
thinking_frame = SpriteFrame(images=thinking_list)
|
||||||
|
|
||||||
|
|
||||||
@@ -115,7 +121,7 @@ async def main(room_url: str, token):
|
|||||||
mic_sample_rate=16000,
|
mic_sample_rate=16000,
|
||||||
camera_enabled=True,
|
camera_enabled=True,
|
||||||
camera_width=720,
|
camera_width=720,
|
||||||
camera_height=1280
|
camera_height=1280,
|
||||||
)
|
)
|
||||||
transport._mic_enabled = True
|
transport._mic_enabled = True
|
||||||
transport._mic_sample_rate = 16000
|
transport._mic_sample_rate = 16000
|
||||||
@@ -126,25 +132,31 @@ async def main(room_url: str, token):
|
|||||||
llm = AzureLLMService(
|
llm = AzureLLMService(
|
||||||
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
api_key=os.getenv("AZURE_CHATGPT_API_KEY"),
|
||||||
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"),
|
||||||
model=os.getenv("AZURE_CHATGPT_MODEL"))
|
model=os.getenv("AZURE_CHATGPT_MODEL"),
|
||||||
|
)
|
||||||
tts = ElevenLabsTTSService(
|
tts = ElevenLabsTTSService(
|
||||||
aiohttp_session=session,
|
aiohttp_session=session,
|
||||||
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
voice_id="jBpfuIE2acCO8z3wKNLl")
|
voice_id="jBpfuIE2acCO8z3wKNLl",
|
||||||
|
)
|
||||||
isa = ImageSyncAggregator()
|
isa = ImageSyncAggregator()
|
||||||
|
|
||||||
@transport.event_handler("on_first_other_participant_joined")
|
@transport.event_handler("on_first_other_participant_joined")
|
||||||
async def on_first_other_participant_joined(transport):
|
async def on_first_other_participant_joined(transport):
|
||||||
await tts.say("Hi! If you want to talk to me, just say 'hey Santa Cat'.", transport.send_queue)
|
await tts.say(
|
||||||
|
"Hi! If you want to talk to me, just say 'hey Santa Cat'.",
|
||||||
|
transport.send_queue,
|
||||||
|
)
|
||||||
|
|
||||||
async def handle_transcriptions():
|
async def handle_transcriptions():
|
||||||
messages = [
|
messages = [
|
||||||
{"role": "system", "content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long."},
|
{
|
||||||
|
"role": "system",
|
||||||
|
"content": "You are Santa Cat, a cat that lives in Santa's workshop at the North Pole. You should be clever, and a bit sarcastic. You should also tell jokes every once in a while. Your responses should only be a few sentences long.",
|
||||||
|
},
|
||||||
]
|
]
|
||||||
|
|
||||||
tma_in = LLMUserContextAggregator(
|
tma_in = LLMUserContextAggregator(messages, transport._my_participant_id)
|
||||||
messages, transport._my_participant_id
|
|
||||||
)
|
|
||||||
tma_out = LLMAssistantContextAggregator(
|
tma_out = LLMAssistantContextAggregator(
|
||||||
messages, transport._my_participant_id
|
messages, transport._my_participant_id
|
||||||
)
|
)
|
||||||
@@ -155,16 +167,10 @@ async def main(room_url: str, token):
|
|||||||
isa.run(
|
isa.run(
|
||||||
tma_out.run(
|
tma_out.run(
|
||||||
llm.run(
|
llm.run(
|
||||||
tma_in.run(
|
tma_in.run(ncf.run(tf.run(transport.get_receive_frames())))
|
||||||
ncf.run(
|
|
||||||
tf.run(
|
|
||||||
transport.get_receive_frames()
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
)
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
async def starting_image():
|
async def starting_image():
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from typing import AsyncGenerator
|
|||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
logger = logging.getLogger("dailyai")
|
logger = logging.getLogger("dailyai")
|
||||||
logger.setLevel(logging.DEBUG)
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
|
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.whisper_ai_services import WhisperSTTService
|
from dailyai.services.whisper_ai_services import WhisperSTTService
|
||||||
|
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url: str):
|
async def main(room_url: str):
|
||||||
transport = DailyTransportService(
|
transport = DailyTransportService(
|
||||||
@@ -14,7 +18,7 @@ async def main(room_url: str):
|
|||||||
start_transcription=True,
|
start_transcription=True,
|
||||||
mic_enabled=False,
|
mic_enabled=False,
|
||||||
camera_enabled=False,
|
camera_enabled=False,
|
||||||
speaker_enabled=True
|
speaker_enabled=True,
|
||||||
)
|
)
|
||||||
|
|
||||||
stt = WhisperSTTService()
|
stt = WhisperSTTService()
|
||||||
@@ -28,9 +32,9 @@ async def main(room_url: str):
|
|||||||
|
|
||||||
async def handle_speaker():
|
async def handle_speaker():
|
||||||
await stt.run_to_queue(
|
await stt.run_to_queue(
|
||||||
transcription_output_queue,
|
transcription_output_queue, transport.get_receive_frames()
|
||||||
transport.get_receive_frames()
|
|
||||||
)
|
)
|
||||||
|
|
||||||
await asyncio.gather(transport.run(), handle_speaker(), handle_transcription())
|
await asyncio.gather(transport.run(), handle_speaker(), handle_transcription())
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,11 +1,16 @@
|
|||||||
import argparse
|
import argparse
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import wave
|
import wave
|
||||||
from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
|
from dailyai.pipeline.frames import EndFrame, TranscriptionQueueFrame
|
||||||
|
|
||||||
from dailyai.services.local_transport_service import LocalTransportService
|
from dailyai.services.local_transport_service import LocalTransportService
|
||||||
from dailyai.services.whisper_ai_services import WhisperSTTService
|
from dailyai.services.whisper_ai_services import WhisperSTTService
|
||||||
|
|
||||||
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url: str):
|
async def main(room_url: str):
|
||||||
global transport
|
global transport
|
||||||
@@ -17,7 +22,7 @@ async def main(room_url: str):
|
|||||||
camera_enabled=False,
|
camera_enabled=False,
|
||||||
speaker_enabled=True,
|
speaker_enabled=True,
|
||||||
duration_minutes=meeting_duration_minutes,
|
duration_minutes=meeting_duration_minutes,
|
||||||
start_transcription=True
|
start_transcription=True,
|
||||||
)
|
)
|
||||||
stt = WhisperSTTService()
|
stt = WhisperSTTService()
|
||||||
transcription_output_queue = asyncio.Queue()
|
transcription_output_queue = asyncio.Queue()
|
||||||
|
|||||||
@@ -2,35 +2,48 @@ import aiohttp
|
|||||||
import asyncio
|
import asyncio
|
||||||
import json
|
import json
|
||||||
import random
|
import random
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import wave
|
import wave
|
||||||
from typing import AsyncGenerator
|
from typing import AsyncGenerator
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
import sys
|
|
||||||
print('\n'.join(sys.path))
|
|
||||||
|
|
||||||
from dailyai.pipeline.pipeline import Pipeline
|
from dailyai.pipeline.pipeline import Pipeline
|
||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
from dailyai.services.open_ai_services import OpenAILLMService
|
from dailyai.services.open_ai_services import OpenAILLMService
|
||||||
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator, UserResponseAggregator, LLMResponseAggregator
|
from dailyai.pipeline.aggregators import (
|
||||||
|
LLMAssistantContextAggregator,
|
||||||
|
LLMContextAggregator,
|
||||||
|
LLMUserContextAggregator,
|
||||||
|
UserResponseAggregator,
|
||||||
|
LLMResponseAggregator,
|
||||||
|
)
|
||||||
from examples.support.runner import configure
|
from examples.support.runner import configure
|
||||||
from dailyai.pipeline.frames import LLMMessagesQueueFrame, TranscriptionQueueFrame, Frame, TextFrame, LLMFunctionCallFrame, LLMFunctionStartFrame, LLMResponseEndFrame, StartFrame, AudioFrame, SpriteFrame, ImageFrame
|
from dailyai.pipeline.frames import (
|
||||||
|
LLMMessagesQueueFrame,
|
||||||
|
TranscriptionQueueFrame,
|
||||||
|
Frame,
|
||||||
|
TextFrame,
|
||||||
|
LLMFunctionCallFrame,
|
||||||
|
LLMFunctionStartFrame,
|
||||||
|
LLMResponseEndFrame,
|
||||||
|
StartFrame,
|
||||||
|
AudioFrame,
|
||||||
|
SpriteFrame,
|
||||||
|
ImageFrame,
|
||||||
|
)
|
||||||
from dailyai.services.ai_services import FrameLogger, AIService
|
from dailyai.services.ai_services import FrameLogger, AIService
|
||||||
|
|
||||||
import logging
|
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
|
||||||
logging.basicConfig(level=logging.INFO)
|
logger = logging.getLogger("dailyai")
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
|
||||||
sounds = {}
|
sounds = {}
|
||||||
sound_files = [
|
sound_files = ["clack-short.wav", "clack.wav", "clack-short-quiet.wav"]
|
||||||
'clack-short.wav',
|
|
||||||
'clack.wav',
|
|
||||||
'clack-short-quiet.wav'
|
|
||||||
]
|
|
||||||
|
|
||||||
script_dir = os.path.dirname(__file__)
|
script_dir = os.path.dirname(__file__)
|
||||||
|
|
||||||
@@ -48,9 +61,11 @@ steps = [
|
|||||||
{
|
{
|
||||||
"prompt": "Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function.",
|
"prompt": "Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function.",
|
||||||
"run_async": False,
|
"run_async": False,
|
||||||
"failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.", "tools": [{
|
"failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.",
|
||||||
"type": "function",
|
"tools": [
|
||||||
"function": {
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
"name": "verify_birthday",
|
"name": "verify_birthday",
|
||||||
"description": "Use this function to verify the user has provided their correct birthday.",
|
"description": "Use this function to verify the user has provided their correct birthday.",
|
||||||
"parameters": {
|
"parameters": {
|
||||||
@@ -58,18 +73,21 @@ steps = [
|
|||||||
"properties": {
|
"properties": {
|
||||||
"birthday": {
|
"birthday": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function."
|
"description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function.",
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}]},
|
],
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"prompt": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages.",
|
"prompt": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages.",
|
||||||
"run_async": True,
|
"run_async": True,
|
||||||
"tools": [{
|
"tools": [
|
||||||
"type": "function",
|
{
|
||||||
"function": {
|
"type": "function",
|
||||||
|
"function": {
|
||||||
"name": "list_prescriptions",
|
"name": "list_prescriptions",
|
||||||
"description": "Once the user has provided a list of their prescription medications, call this function.",
|
"description": "Once the user has provided a list of their prescription medications, call this function.",
|
||||||
"parameters": {
|
"parameters": {
|
||||||
@@ -82,19 +100,20 @@ steps = [
|
|||||||
"properties": {
|
"properties": {
|
||||||
"medication": {
|
"medication": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "The medication's name"
|
"description": "The medication's name",
|
||||||
},
|
},
|
||||||
"dosage": {
|
"dosage": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "The prescription's dosage"
|
"description": "The prescription's dosage",
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}]
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"prompt": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function.",
|
"prompt": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function.",
|
||||||
@@ -115,16 +134,16 @@ steps = [
|
|||||||
"properties": {
|
"properties": {
|
||||||
"name": {
|
"name": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "What the user is allergic to"
|
"description": "What the user is allergic to",
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"prompt": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function.",
|
"prompt": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function.",
|
||||||
@@ -145,14 +164,14 @@ steps = [
|
|||||||
"properties": {
|
"properties": {
|
||||||
"name": {
|
"name": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "The user's medical condition"
|
"description": "The user's medical condition",
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
},
|
},
|
||||||
@@ -175,20 +194,23 @@ steps = [
|
|||||||
"properties": {
|
"properties": {
|
||||||
"name": {
|
"name": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"description": "The user's reason for visiting the doctor"
|
"description": "The user's reason for visiting the doctor",
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
}
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
},
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
},
|
},
|
||||||
{"prompt": "Now, thank the user and end the conversation.",
|
{
|
||||||
"run_async": True, "tools": []},
|
"prompt": "Now, thank the user and end the conversation.",
|
||||||
{"prompt": "", "run_async": True, "tools": []}
|
"run_async": True,
|
||||||
|
"tools": [],
|
||||||
|
},
|
||||||
|
{"prompt": "", "run_async": True, "tools": []},
|
||||||
]
|
]
|
||||||
current_step = 0
|
current_step = 0
|
||||||
|
|
||||||
@@ -219,15 +241,15 @@ class ChecklistProcessor(AIService):
|
|||||||
"list_prescriptions",
|
"list_prescriptions",
|
||||||
"list_allergies",
|
"list_allergies",
|
||||||
"list_conditions",
|
"list_conditions",
|
||||||
"list_visit_reasons"
|
"list_visit_reasons",
|
||||||
|
|
||||||
]
|
]
|
||||||
|
|
||||||
messages.append(
|
messages.append(
|
||||||
{"role": "system", "content": f"{self._id} {steps[0]['prompt']}"})
|
{"role": "system", "content": f"{self._id} {steps[0]['prompt']}"}
|
||||||
|
)
|
||||||
|
|
||||||
def verify_birthday(self, args):
|
def verify_birthday(self, args):
|
||||||
return args['birthday'] == "1983-01-01"
|
return args["birthday"] == "1983-01-01"
|
||||||
|
|
||||||
def list_prescriptions(self, args):
|
def list_prescriptions(self, args):
|
||||||
# print(f"--- Prescriptions: {args['prescriptions']}\n")
|
# print(f"--- Prescriptions: {args['prescriptions']}\n")
|
||||||
@@ -250,18 +272,21 @@ class ChecklistProcessor(AIService):
|
|||||||
this_step = steps[current_step]
|
this_step = steps[current_step]
|
||||||
# TODO-CB: forcing a global here :/
|
# TODO-CB: forcing a global here :/
|
||||||
self._tools.clear()
|
self._tools.clear()
|
||||||
self._tools.extend(this_step['tools'])
|
self._tools.extend(this_step["tools"])
|
||||||
if isinstance(frame, LLMFunctionStartFrame):
|
if isinstance(frame, LLMFunctionStartFrame):
|
||||||
print(f"... Preparing function call: {frame.function_name}")
|
print(f"... Preparing function call: {frame.function_name}")
|
||||||
self._function_name = frame.function_name
|
self._function_name = frame.function_name
|
||||||
if this_step['run_async']:
|
if this_step["run_async"]:
|
||||||
# Get the LLM talking about the next step before getting the rest
|
# Get the LLM talking about the next step before getting the rest
|
||||||
# of the function call completion
|
# of the function call completion
|
||||||
current_step += 1
|
current_step += 1
|
||||||
self._messages.append({
|
self._messages.append(
|
||||||
"role": "system", "content": steps[current_step]['prompt']})
|
{"role": "system", "content": steps[current_step]["prompt"]}
|
||||||
|
)
|
||||||
yield LLMMessagesQueueFrame(self._messages)
|
yield LLMMessagesQueueFrame(self._messages)
|
||||||
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
|
async for frame in llm.process_frame(
|
||||||
|
LLMMessagesQueueFrame(self._messages), tool_choice="none"
|
||||||
|
):
|
||||||
yield frame
|
yield frame
|
||||||
else:
|
else:
|
||||||
# Insert a quick response while we run the function
|
# Insert a quick response while we run the function
|
||||||
@@ -270,29 +295,37 @@ class ChecklistProcessor(AIService):
|
|||||||
elif isinstance(frame, LLMFunctionCallFrame):
|
elif isinstance(frame, LLMFunctionCallFrame):
|
||||||
|
|
||||||
if frame.function_name and frame.arguments:
|
if frame.function_name and frame.arguments:
|
||||||
print(
|
print(f"--> Calling function: {frame.function_name} with arguments:")
|
||||||
f"--> Calling function: {frame.function_name} with arguments:")
|
pretty_json = re.sub(
|
||||||
pretty_json = re.sub("\n", "\n ", json.dumps(
|
"\n", "\n ", json.dumps(json.loads(frame.arguments), indent=2)
|
||||||
json.loads(frame.arguments), indent=2))
|
)
|
||||||
print(f"--> {pretty_json}\n")
|
print(f"--> {pretty_json}\n")
|
||||||
if not frame.function_name in self._functions:
|
if not frame.function_name in self._functions:
|
||||||
raise Exception(f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions.")
|
raise Exception(
|
||||||
|
f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions."
|
||||||
|
)
|
||||||
fn = getattr(self, frame.function_name)
|
fn = getattr(self, frame.function_name)
|
||||||
result = fn(json.loads(frame.arguments))
|
result = fn(json.loads(frame.arguments))
|
||||||
|
|
||||||
if not this_step['run_async']:
|
if not this_step["run_async"]:
|
||||||
if result:
|
if result:
|
||||||
current_step += 1
|
current_step += 1
|
||||||
self._messages.append({
|
self._messages.append(
|
||||||
"role": "system", "content": steps[current_step]['prompt']})
|
{"role": "system", "content": steps[current_step]["prompt"]}
|
||||||
|
)
|
||||||
yield LLMMessagesQueueFrame(self._messages)
|
yield LLMMessagesQueueFrame(self._messages)
|
||||||
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
|
async for frame in llm.process_frame(
|
||||||
|
LLMMessagesQueueFrame(self._messages), tool_choice="none"
|
||||||
|
):
|
||||||
yield frame
|
yield frame
|
||||||
else:
|
else:
|
||||||
self._messages.append({
|
self._messages.append(
|
||||||
"role": "system", "content": this_step['failed']})
|
{"role": "system", "content": this_step["failed"]}
|
||||||
|
)
|
||||||
yield LLMMessagesQueueFrame(self._messages)
|
yield LLMMessagesQueueFrame(self._messages)
|
||||||
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
|
async for frame in llm.process_frame(
|
||||||
|
LLMMessagesQueueFrame(self._messages), tool_choice="none"
|
||||||
|
):
|
||||||
yield frame
|
yield frame
|
||||||
print(f"<-- Verify result: {result}\n")
|
print(f"<-- Verify result: {result}\n")
|
||||||
|
|
||||||
@@ -315,7 +348,7 @@ async def main(room_url: str, token):
|
|||||||
mic_sample_rate=16000,
|
mic_sample_rate=16000,
|
||||||
camera_enabled=False,
|
camera_enabled=False,
|
||||||
start_transcription=True,
|
start_transcription=True,
|
||||||
vad_enabled=True
|
vad_enabled=True,
|
||||||
)
|
)
|
||||||
# TODO-CB: Go back to vad_enabled
|
# TODO-CB: Go back to vad_enabled
|
||||||
|
|
||||||
@@ -324,12 +357,18 @@ async def main(room_url: str, token):
|
|||||||
|
|
||||||
# llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv(
|
# llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv(
|
||||||
# "AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
|
# "AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
|
||||||
llm = OpenAILLMService(api_key=os.getenv(
|
llm = OpenAILLMService(
|
||||||
"OPENAI_CHATGPT_API_KEY"), model="gpt-4-1106-preview", tools=tools) # gpt-4-1106-preview
|
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
|
||||||
|
model="gpt-4-1106-preview",
|
||||||
|
tools=tools,
|
||||||
|
) # gpt-4-1106-preview
|
||||||
# tts = AzureTTSService(api_key=os.getenv(
|
# tts = AzureTTSService(api_key=os.getenv(
|
||||||
# "AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
|
# "AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
|
||||||
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv(
|
tts = ElevenLabsTTSService(
|
||||||
"ELEVENLABS_API_KEY"), voice_id="XrExE9yKIg1WjnnlVkGX") # matilda
|
aiohttp_session=session,
|
||||||
|
api_key=os.getenv("ELEVENLABS_API_KEY"),
|
||||||
|
voice_id="XrExE9yKIg1WjnnlVkGX",
|
||||||
|
) # matilda
|
||||||
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv(
|
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv(
|
||||||
# "DEEPGRAM_API_KEY"), voice="aura-asteria-en")
|
# "DEEPGRAM_API_KEY"), voice="aura-asteria-en")
|
||||||
|
|
||||||
@@ -345,34 +384,23 @@ async def main(room_url: str, token):
|
|||||||
# TODO-CB: Make sure this message gets into the context somehow
|
# TODO-CB: Make sure this message gets into the context somehow
|
||||||
await tts.run_to_queue(
|
await tts.run_to_queue(
|
||||||
transport.send_queue,
|
transport.send_queue,
|
||||||
llm.run([LLMMessagesQueueFrame(messages)]),
|
llm.run([LLMMessagesQueueFrame(messages)]),
|
||||||
|
|
||||||
)
|
)
|
||||||
|
|
||||||
async def handle_intake():
|
async def handle_intake():
|
||||||
pipeline = Pipeline(
|
pipeline = Pipeline(processors=[fl, llm, fl2, checklist, tts])
|
||||||
processors=[
|
await transport.run_interruptible_pipeline(
|
||||||
fl,
|
pipeline,
|
||||||
llm,
|
post_processor=LLMResponseAggregator(messages),
|
||||||
fl2,
|
pre_processor=UserResponseAggregator(messages),
|
||||||
checklist,
|
|
||||||
tts
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
await transport.run_interruptible_pipeline(pipeline,
|
|
||||||
post_processor=LLMResponseAggregator(
|
|
||||||
messages
|
|
||||||
),
|
|
||||||
pre_processor=UserResponseAggregator(messages)
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
transport.transcription_settings["extra"]["endpointing"] = True
|
transport.transcription_settings["extra"]["endpointing"] = True
|
||||||
transport.transcription_settings["extra"]["punctuate"] = True
|
transport.transcription_settings["extra"]["punctuate"] = True
|
||||||
try:
|
try:
|
||||||
await asyncio.gather(transport.run(), handle_intake())
|
await asyncio.gather(transport.run(), handle_intake())
|
||||||
except (asyncio.CancelledError, KeyboardInterrupt):
|
except (asyncio.CancelledError, KeyboardInterrupt):
|
||||||
print('whoops')
|
print("whoops")
|
||||||
transport.stop()
|
transport.stop()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user