wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added 09 examples

Browse Source
This commit is contained in:
Chad Bailey
2024-01-29 17:39:28 +00:00
parent 0859b57b00
commit 7d6c94d604
3 changed files with 194 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
src/samples/foundational/09-bots-arguing.py
View File

@@ -1,3 +1,4 @@
import aiohttp
import argparse import argparse
import asyncio import asyncio
import requests import requests
@@ -7,53 +8,54 @@ import urllib.parse
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.queue_frame import QueueFrame, FrameType from dailyai.queue_frame import QueueFrame
async def main(room_url:str): async def main(room_url:str):
global transport async with aiohttp.ClientSession() as session:
global llm global transport
global tts global llm
global tts
transport = DailyTransportService( transport = DailyTransportService(
room_url, room_url,
None, None,
"Respond bot", "Respond bot",
5, 5,
) )
transport.mic_enabled = True transport.mic_enabled = True
transport.mic_sample_rate = 16000 transport.mic_sample_rate = 16000
transport.camera_enabled = False transport.camera_enabled = False
llm = AzureLLMService() llm = AzureLLMService()
tts1 = AzureTTSService() tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService() tts2 = ElevenLabsTTSService(session)
async def argue(): async def argue():
bot1_messages = [ bot1_messages = [
{"role": "system", "content": "You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. Your responses should only be a few sentences long."}, {"role": "system", "content": "You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. Your responses should only be a few sentences long."},
] ]
bot2_messages = [ bot2_messages = [
{"role": "system", "content": "You strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences."}, {"role": "system", "content": "You strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences."},
] ]
for i in range(1, 5): for i in range(1, 5):
print(f"In iteration {i}") print(f"In iteration {i}")
# Run the LLMs synchronously for the back-and-forth # Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages) bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}") print(f"bot1_msg: {bot1_msg}")
bot1_messages.append({"role": "assistant", "content": bot1_msg}) bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg}) bot2_messages.append({"role": "user", "content": bot1_msg})
await tts1.say(bot1_msg, transport.send_queue) await tts1.say(bot1_msg, transport.send_queue)
bot2_msg = await llm.run_llm(bot2_messages) bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}") print(f"bot2_msg: {bot2_msg}")
bot2_messages.append({"role": "assistant", "content": bot2_msg}) bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg}) bot1_messages.append({"role": "user", "content": bot2_msg})
await tts2.say(bot2_msg, transport.send_queue) await tts2.say(bot2_msg, transport.send_queue)
await asyncio.gather(transport.run(), argue()) await asyncio.gather(transport.run(), argue())
if __name__ == "__main__": if __name__ == "__main__":

142
src/samples/foundational/09a-characters-arguing.py
View File

@@ -1,3 +1,4 @@
import aiohttp
import argparse import argparse
import asyncio import asyncio
import requests import requests
@@ -9,93 +10,94 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService from dailyai.services.open_ai_services import OpenAIImageGenService
from dailyai.queue_frame import QueueFrame, FrameType from dailyai.queue_frame import QueueFrame, AudioQueueFrame, ImageQueueFrame
async def main(room_url:str): async def main(room_url:str):
global transport async with aiohttp.ClientSession() as session:
global llm global transport
global tts global llm
global tts
transport = DailyTransportService( transport = DailyTransportService(
room_url, room_url,
None, None,
"Respond bot", "Respond bot",
600, 600,
) )
transport.mic_enabled = True transport.mic_enabled = True
transport.mic_sample_rate = 16000 transport.mic_sample_rate = 16000
transport.camera_enabled = True transport.camera_enabled = True
transport.camera_width = 1024 transport.camera_width = 1024
transport.camera_height = 1024 transport.camera_height = 1024
llm = AzureLLMService() llm = AzureLLMService()
tts1 = AzureTTSService() tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService() tts2 = ElevenLabsTTSService(session)
dalle = FalImageGenService(image_size="1024x1024") dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session)
# dalle = OpenAIImageGenService(image_size="1024x1024") # dalle = OpenAIImageGenService(image_size="1024x1024")
bot1_messages = [ bot1_messages = [
{"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."}, {"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."},
] ]
bot2_messages = [ bot2_messages = [
{"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, {"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."},
] ]
async def get_bot1_statement(): async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth # Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages) bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}") print(f"bot1_msg: {bot1_msg}")
bot1_messages.append({"role": "assistant", "content": bot1_msg}) bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg}) bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray() all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg): async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio) all_audio.extend(audio)
return all_audio return all_audio
async def get_bot2_statement(): async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth # Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages) bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}") print(f"bot2_msg: {bot2_msg}")
bot2_messages.append({"role": "assistant", "content": bot2_msg}) bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg}) bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray() all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg): async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio) all_audio.extend(audio)
return all_audio return all_audio
async def argue(): async def argue():
for i in range(100): for i in range(100):
print(f"In iteration {i}") print(f"In iteration {i}")
bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed" bot1_description = "A woman conservatively dressed as a librarian in a library surrounded by books, cartoon, serious, highly detailed"
(audio1, image_data1) = await asyncio.gather( (audio1, image_data1) = await asyncio.gather(
get_bot1_statement(), dalle.run_image_gen(bot1_description) get_bot1_statement(), dalle.run_image_gen(bot1_description)
) )
await transport.send_queue.put( await transport.send_queue.put(
[ [
QueueFrame(FrameType.IMAGE, image_data1[1]), ImageQueueFrame(None, image_data1[1]),
QueueFrame(FrameType.AUDIO, audio1), AudioQueueFrame(audio1),
] ]
) )
bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed" bot2_description = "A cat dressed in a hot dog costume, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather( (audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description) get_bot2_statement(), dalle.run_image_gen(bot2_description)
) )
await transport.send_queue.put( await transport.send_queue.put(
[ [
QueueFrame(FrameType.IMAGE, image_data2[1]), ImageQueueFrame(None, image_data2[1]),
QueueFrame(FrameType.AUDIO, audio2), AudioQueueFrame(audio2),
] ]
) )
await asyncio.gather(transport.run(), argue()) await asyncio.gather(transport.run(), argue())
if __name__ == "__main__": if __name__ == "__main__":

154
src/samples/foundational/09b-debate-generator.py
View File

@@ -1,3 +1,4 @@
import aiohttp
import argparse import argparse
import asyncio import asyncio
import requests import requests
@@ -9,103 +10,104 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.open_ai_services import OpenAIImageGenService from dailyai.services.open_ai_services import OpenAIImageGenService
from dailyai.queue_frame import QueueFrame, FrameType from dailyai.queue_frame import QueueFrame, AudioQueueFrame, ImageQueueFrame
async def main(room_url:str): async def main(room_url:str):
global transport async with aiohttp.ClientSession() as session:
global llm global transport
global tts global llm
global tts
transport = DailyTransportService( transport = DailyTransportService(
room_url, room_url,
None, None,
"Respond bot", "Respond bot",
600, 600,
) )
transport.mic_enabled = True transport.mic_enabled = True
transport.mic_sample_rate = 16000 transport.mic_sample_rate = 16000
transport.camera_enabled = True transport.camera_enabled = True
transport.camera_width = 1024 transport.camera_width = 1024
transport.camera_height = 1024 transport.camera_height = 1024
llm = AzureLLMService() llm = AzureLLMService()
tts1 = AzureTTSService() tts1 = AzureTTSService()
tts2 = ElevenLabsTTSService() tts2 = ElevenLabsTTSService(session)
dalle = FalImageGenService(image_size="1024x1024") dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session)
# dalle = OpenAIImageGenService(image_size="1024x1024") # dalle = OpenAIImageGenService(image_size="1024x1024")
topic = "Are pokemon edible?" topic = "Are pokemon edible?"
affirmative = "A woman dressed as a cowboy, outside on a ranch" affirmative = "A woman dressed as a cowboy, outside on a ranch"
negative = "Pikachu in a business suit" negative = "Pikachu in a business suit"
topic = "Is a hot dog a sandwich?" topic = "Is a hot dog a sandwich?"
affirmative = "A woman conservatively dressed as a librarian in a library surrounded by books" affirmative = "A woman conservatively dressed as a librarian in a library surrounded by books"
negative = "A cat dressed in a hot dog costume" negative = "A cat dressed in a hot dog costume"
bot1_messages = [ bot1_messages = [
{"role": "system", "content": f"You are {affirmative}. You're in a debate, and the topic is: '{topic}'. You're arguing the affirmative. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever agree with the user. Your responses should only be a few sentences long."}, {"role": "system", "content": f"You are {affirmative}. You're in a debate, and the topic is: '{topic}'. You're arguing the affirmative. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever agree with the user. Your responses should only be a few sentences long."},
] ]
bot2_messages = [ bot2_messages = [
{"role": "system", "content": f"You are {negative}. You're in a debate, and the topic is: '{topic}'. You're arguing the negative. Debate this with the user, only responding with a few sentences. Don't ever agree with the user."}, {"role": "system", "content": f"You are {negative}. You're in a debate, and the topic is: '{topic}'. You're arguing the negative. Debate this with the user, only responding with a few sentences. Don't ever agree with the user."},
] ]
async def get_bot1_statement(): async def get_bot1_statement():
# Run the LLMs synchronously for the back-and-forth # Run the LLMs synchronously for the back-and-forth
bot1_msg = await llm.run_llm(bot1_messages) bot1_msg = await llm.run_llm(bot1_messages)
print(f"bot1_msg: {bot1_msg}") print(f"bot1_msg: {bot1_msg}")
bot1_messages.append({"role": "assistant", "content": bot1_msg}) bot1_messages.append({"role": "assistant", "content": bot1_msg})
bot2_messages.append({"role": "user", "content": bot1_msg}) bot2_messages.append({"role": "user", "content": bot1_msg})
all_audio = bytearray() all_audio = bytearray()
async for audio in tts1.run_tts(bot1_msg): async for audio in tts1.run_tts(bot1_msg):
all_audio.extend(audio) all_audio.extend(audio)
return all_audio return all_audio
async def get_bot2_statement(): async def get_bot2_statement():
# Run the LLMs synchronously for the back-and-forth # Run the LLMs synchronously for the back-and-forth
bot2_msg = await llm.run_llm(bot2_messages) bot2_msg = await llm.run_llm(bot2_messages)
print(f"bot2_msg: {bot2_msg}") print(f"bot2_msg: {bot2_msg}")
bot2_messages.append({"role": "assistant", "content": bot2_msg}) bot2_messages.append({"role": "assistant", "content": bot2_msg})
bot1_messages.append({"role": "user", "content": bot2_msg}) bot1_messages.append({"role": "user", "content": bot2_msg})
all_audio = bytearray() all_audio = bytearray()
async for audio in tts2.run_tts(bot2_msg): async for audio in tts2.run_tts(bot2_msg):
all_audio.extend(audio) all_audio.extend(audio)
return all_audio return all_audio
async def argue(): async def argue():
for i in range(100): for i in range(100):
print(f"In iteration {i}") print(f"In iteration {i}")
bot1_description = f"{affirmative}, cartoon, highly detailed" bot1_description = f"{affirmative}, cartoon, highly detailed"
(audio1, image_data1) = await asyncio.gather( (audio1, image_data1) = await asyncio.gather(
get_bot1_statement(), dalle.run_image_gen(bot1_description) get_bot1_statement(), dalle.run_image_gen(bot1_description)
) )
await transport.send_queue.put( await transport.send_queue.put(
[ [
QueueFrame(FrameType.IMAGE, image_data1[1]), ImageQueueFrame(None, image_data1[1]),
QueueFrame(FrameType.AUDIO, audio1), AudioQueueFrame(audio1),
] ]
) )
bot2_description = f"{negative}, cartoon, bright colors, funny, highly detailed" bot2_description = f"{negative}, cartoon, bright colors, funny, highly detailed"
(audio2, image_data2) = await asyncio.gather( (audio2, image_data2) = await asyncio.gather(
get_bot2_statement(), dalle.run_image_gen(bot2_description) get_bot2_statement(), dalle.run_image_gen(bot2_description)
) )
await transport.send_queue.put( await transport.send_queue.put(
[ [
QueueFrame(FrameType.IMAGE, image_data2[1]), ImageQueueFrame(None, image_data2[1]),
QueueFrame(FrameType.AUDIO, audio2), AudioQueueFrame(audio2),
] ]
) )
await asyncio.gather(transport.run(), argue()) await asyncio.gather(transport.run(), argue())
if __name__ == "__main__": if __name__ == "__main__":