wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleaned up example logging (#46)

Browse Source
This commit is contained in:
chadbailey59
2024-03-08 15:25:17 -06:00
committed by GitHub
parent 95a1efbe75
commit 8241dc0bed
20 changed files with 410 additions and 274 deletions
Download Patch File Download Diff File Expand all files Collapse all files

226
src/examples/starter-apps/patient-intake.py
View File

@@ -2,35 +2,48 @@ import aiohttp
import asyncio
import json
import random
import logging
import os
import re
import wave
from typing import AsyncGenerator
from PIL import Image
import sys
print('\n'.join(sys.path))
from dailyai.pipeline.pipeline import Pipeline
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.open_ai_services import OpenAILLMService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.pipeline.aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator, UserResponseAggregator, LLMResponseAggregator
from dailyai.pipeline.aggregators import (
LLMAssistantContextAggregator,
LLMContextAggregator,
LLMUserContextAggregator,
UserResponseAggregator,
LLMResponseAggregator,
)
from examples.support.runner import configure
from dailyai.pipeline.frames import LLMMessagesQueueFrame, TranscriptionQueueFrame, Frame, TextFrame, LLMFunctionCallFrame, LLMFunctionStartFrame, LLMResponseEndFrame, StartFrame, AudioFrame, SpriteFrame, ImageFrame
from dailyai.pipeline.frames import (
LLMMessagesQueueFrame,
TranscriptionQueueFrame,
Frame,
TextFrame,
LLMFunctionCallFrame,
LLMFunctionStartFrame,
LLMResponseEndFrame,
StartFrame,
AudioFrame,
SpriteFrame,
ImageFrame,
)
from dailyai.services.ai_services import FrameLogger, AIService
import logging
logging.basicConfig(level=logging.INFO)
logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s")
logger = logging.getLogger("dailyai")
logger.setLevel(logging.DEBUG)
sounds = {}
sound_files = [
'clack-short.wav',
'clack.wav',
'clack-short-quiet.wav'
]
sound_files = ["clack-short.wav", "clack.wav", "clack-short-quiet.wav"]
script_dir = os.path.dirname(__file__)
@@ -48,9 +61,11 @@ steps = [
{
"prompt": "Start by introducing yourself. Then, ask the user to confirm their identity by telling you their birthday, including the year. When they answer with their birthday, call the verify_birthday function.",
"run_async": False,
"failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.", "tools": [{
"type": "function",
"function": {
"failed": "The user provided an incorrect birthday. Ask them for their birthday again. When they answer, call the verify_birthday function.",
"tools": [
{
"type": "function",
"function": {
"name": "verify_birthday",
"description": "Use this function to verify the user has provided their correct birthday.",
"parameters": {
@@ -58,18 +73,21 @@ steps = [
"properties": {
"birthday": {
"type": "string",
"description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function."
"description": "The user's birthdate, including the year. The user can provide it in any format, but convert it to YYYY-MM-DD format to call this function.",
}
}
}
},
},
},
}
}]},
],
},
{
"prompt": "Next, thank the user for confirming their identity, then ask the user to list their current prescriptions. Each prescription needs to have a medication name and a dosage. Do not call the list_prescriptions function with any unknown dosages.",
"run_async": True,
"tools": [{
"type": "function",
"function": {
"tools": [
{
"type": "function",
"function": {
"name": "list_prescriptions",
"description": "Once the user has provided a list of their prescription medications, call this function.",
"parameters": {
@@ -82,19 +100,20 @@ steps = [
"properties": {
"medication": {
"type": "string",
"description": "The medication's name"
"description": "The medication's name",
},
"dosage": {
"type": "string",
"description": "The prescription's dosage"
}
}
}
"description": "The prescription's dosage",
},
},
},
}
}
}
},
},
},
}
}]
],
},
{
"prompt": "Next, ask the user if they have any allergies. Once they have listed their allergies or confirmed they don't have any, call the list_allergies function.",
@@ -115,16 +134,16 @@ steps = [
"properties": {
"name": {
"type": "string",
"description": "What the user is allergic to"
"description": "What the user is allergic to",
}
}
}
},
},
}
}
}
}
},
},
},
}
]
],
},
{
"prompt": "Now ask the user if they have any medical conditions the doctor should know about. Once they've answered the question, call the list_conditions function.",
@@ -145,14 +164,14 @@ steps = [
"properties": {
"name": {
"type": "string",
"description": "The user's medical condition"
"description": "The user's medical condition",
}
}
}
},
},
}
}
}
}
},
},
},
},
],
},
@@ -175,20 +194,23 @@ steps = [
"properties": {
"name": {
"type": "string",
"description": "The user's reason for visiting the doctor"
"description": "The user's reason for visiting the doctor",
}
}
}
},
},
}
}
}
}
},
},
},
}
]
],
},
{"prompt": "Now, thank the user and end the conversation.",
"run_async": True, "tools": []},
{"prompt": "", "run_async": True, "tools": []}
{
"prompt": "Now, thank the user and end the conversation.",
"run_async": True,
"tools": [],
},
{"prompt": "", "run_async": True, "tools": []},
]
current_step = 0
@@ -219,15 +241,15 @@ class ChecklistProcessor(AIService):
"list_prescriptions",
"list_allergies",
"list_conditions",
"list_visit_reasons"
"list_visit_reasons",
]
messages.append(
{"role": "system", "content": f"{self._id} {steps[0]['prompt']}"})
{"role": "system", "content": f"{self._id} {steps[0]['prompt']}"}
)
def verify_birthday(self, args):
return args['birthday'] == "1983-01-01"
return args["birthday"] == "1983-01-01"
def list_prescriptions(self, args):
# print(f"--- Prescriptions: {args['prescriptions']}\n")
@@ -250,18 +272,21 @@ class ChecklistProcessor(AIService):
this_step = steps[current_step]
# TODO-CB: forcing a global here :/
self._tools.clear()
self._tools.extend(this_step['tools'])
self._tools.extend(this_step["tools"])
if isinstance(frame, LLMFunctionStartFrame):
print(f"... Preparing function call: {frame.function_name}")
self._function_name = frame.function_name
if this_step['run_async']:
if this_step["run_async"]:
# Get the LLM talking about the next step before getting the rest
# of the function call completion
current_step += 1
self._messages.append({
"role": "system", "content": steps[current_step]['prompt']})
self._messages.append(
{"role": "system", "content": steps[current_step]["prompt"]}
)
yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame
else:
# Insert a quick response while we run the function
@@ -270,29 +295,37 @@ class ChecklistProcessor(AIService):
elif isinstance(frame, LLMFunctionCallFrame):
if frame.function_name and frame.arguments:
print(
f"--> Calling function: {frame.function_name} with arguments:")
pretty_json = re.sub("\n", "\n ", json.dumps(
json.loads(frame.arguments), indent=2))
print(f"--> Calling function: {frame.function_name} with arguments:")
pretty_json = re.sub(
"\n", "\n ", json.dumps(json.loads(frame.arguments), indent=2)
)
print(f"--> {pretty_json}\n")
if not frame.function_name in self._functions:
raise Exception(f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions.")
raise Exception(
f"The LLM tried to call a function named {frame.function_name}, which isn't in the list of known functions. Please check your prompt and/or self._functions."
)
fn = getattr(self, frame.function_name)
result = fn(json.loads(frame.arguments))
if not this_step['run_async']:
if not this_step["run_async"]:
if result:
current_step += 1
self._messages.append({
"role": "system", "content": steps[current_step]['prompt']})
self._messages.append(
{"role": "system", "content": steps[current_step]["prompt"]}
)
yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame
else:
self._messages.append({
"role": "system", "content": this_step['failed']})
self._messages.append(
{"role": "system", "content": this_step["failed"]}
)
yield LLMMessagesQueueFrame(self._messages)
async for frame in llm.process_frame(LLMMessagesQueueFrame(self._messages), tool_choice="none"):
async for frame in llm.process_frame(
LLMMessagesQueueFrame(self._messages), tool_choice="none"
):
yield frame
print(f"<-- Verify result: {result}\n")
@@ -315,7 +348,7 @@ async def main(room_url: str, token):
mic_sample_rate=16000,
camera_enabled=False,
start_transcription=True,
vad_enabled=True
vad_enabled=True,
)
# TODO-CB: Go back to vad_enabled
@@ -324,12 +357,18 @@ async def main(room_url: str, token):
# llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv(
# "AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL"))
llm = OpenAILLMService(api_key=os.getenv(
"OPENAI_CHATGPT_API_KEY"), model="gpt-4-1106-preview", tools=tools) # gpt-4-1106-preview
llm = OpenAILLMService(
api_key=os.getenv("OPENAI_CHATGPT_API_KEY"),
model="gpt-4-1106-preview",
tools=tools,
) # gpt-4-1106-preview
# tts = AzureTTSService(api_key=os.getenv(
# "AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION"))
tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv(
"ELEVENLABS_API_KEY"), voice_id="XrExE9yKIg1WjnnlVkGX") # matilda
tts = ElevenLabsTTSService(
aiohttp_session=session,
api_key=os.getenv("ELEVENLABS_API_KEY"),
voice_id="XrExE9yKIg1WjnnlVkGX",
) # matilda
# tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv(
# "DEEPGRAM_API_KEY"), voice="aura-asteria-en")
@@ -345,34 +384,23 @@ async def main(room_url: str, token):
# TODO-CB: Make sure this message gets into the context somehow
await tts.run_to_queue(
transport.send_queue,
llm.run([LLMMessagesQueueFrame(messages)]),
llm.run([LLMMessagesQueueFrame(messages)]),
)
async def handle_intake():
pipeline = Pipeline(
processors=[
fl,
llm,
fl2,
checklist,
tts
]
pipeline = Pipeline(processors=[fl, llm, fl2, checklist, tts])
await transport.run_interruptible_pipeline(
pipeline,
post_processor=LLMResponseAggregator(messages),
pre_processor=UserResponseAggregator(messages),
)
await transport.run_interruptible_pipeline(pipeline,
post_processor=LLMResponseAggregator(
messages
),
pre_processor=UserResponseAggregator(messages)
)
transport.transcription_settings["extra"]["endpointing"] = True
transport.transcription_settings["extra"]["punctuate"] = True
try:
await asyncio.gather(transport.run(), handle_intake())
except (asyncio.CancelledError, KeyboardInterrupt):
print('whoops')
print("whoops")
transport.stop()