wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Adding queue transportation to services

Browse Source
This commit is contained in:
Moishe Lettvin
2024-01-11 19:14:19 -05:00
parent 7ca7764be3
commit b9b82695c6
18 changed files with 194 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
src/samples/static-sprite/sprite-sample.py
View File

@@ -15,7 +15,7 @@ from dailyai.async_processor.async_processor import (
OrchestratorResponse
)
from dailyai.orchestrator import OrchestratorConfig, Orchestrator
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.message_handler.message_handler import MessageHandler
from dailyai.services.ai_services import AIServiceConfig
from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
@@ -40,7 +40,7 @@ class StaticSpriteResponse(OrchestratorResponse):
self.image_bytes = img.tobytes()
def do_play(self) -> None:
self.output_queue.put(OutputQueueFrame(FrameType.IMAGE_FRAME, self.image_bytes))
self.output_queue.put(QueueFrame(FrameType.IMAGE_FRAME, self.image_bytes))
class IntroSpriteResponse(StaticSpriteResponse):
@@ -71,10 +71,10 @@ class AnimatedSpriteLLMResponse(LLMResponse):
with Image.open(full_path) as img:
self.image_bytes.append(img.tobytes())
def get_frames_from_tts_response(self, audio_frame) -> list[OutputQueueFrame]:
def get_frames_from_tts_response(self, audio_frame) -> list[QueueFrame]:
return [
OutputQueueFrame(FrameType.AUDIO_FRAME, audio_frame),
OutputQueueFrame(FrameType.IMAGE_FRAME, random.choice(self.image_bytes))
QueueFrame(FrameType.AUDIO_FRAME, audio_frame),
QueueFrame(FrameType.IMAGE_FRAME, random.choice(self.image_bytes))
]

4
src/samples/theoretical-to-real/01-say-one-thing.py
View File

@@ -2,7 +2,7 @@ import argparse
import asyncio
from typing import AsyncGenerator
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureTTSService
@@ -37,7 +37,7 @@ async def main(room_url):
if participant["info"]["isLocal"]:
return
async for audio in audio_generator:
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio))
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()

6
src/samples/theoretical-to-real/01a-greet-user.py
View File

@@ -2,7 +2,7 @@ import asyncio
import time
from typing import AsyncGenerator
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService
@@ -41,13 +41,13 @@ async def main(room_url):
audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(f"Hello there, {participant['info']['userName']}!")
async for audio in audio_generator:
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio))
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
print("setting up call state handler")
@transport.event_handler("on_call_state_updated")
async def on_call_joined(transport, state):
print(f"call state callback: {state}")
await transport.run()

35
src/samples/theoretical-to-real/02-llm-say-one-thing.py
View File

@@ -1,9 +1,8 @@
import argparse
import asyncio
import re
from typing import AsyncGenerator
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
@@ -21,27 +20,32 @@ async def main(room_url):
)
transport.mic_enabled = True
tts = ElevenLabsTTSService()
llm = AzureLLMService()
text_to_llm_queue = asyncio.Queue()
llm_to_tts_queue = asyncio.Queue()
tts = ElevenLabsTTSService(
llm_to_tts_queue, transport.get_async_output_queue(), voice_id="29vD33N1CtxCmqQRPOHJ"
)
llm = AzureLLMService(text_to_llm_queue, llm_to_tts_queue)
messages = [{
"role": "system",
"content": "You are an LLM in a WebRTC session, and your text will be converted to audio. Introduce yourself."
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world."
}]
llm_generator: AsyncGenerator[str, None] = llm.run_llm_async(messages)
await text_to_llm_queue.put(QueueFrame(FrameType.LLM_MESSAGE_FRAME, messages))
await text_to_llm_queue.put(QueueFrame(FrameType.END_STREAM, None))
llm_task = asyncio.create_task(llm.run())
has_joined = False
@transport.event_handler("on_participant_joined")
async def on_participant_joined(transport, participant):
if participant["id"] == transport.my_participant_id:
nonlocal has_joined
if participant["id"] == transport.my_participant_id or has_joined:
return
current_text = ""
async for text in llm_generator:
current_text += text
if re.match(r"^.*[.!?]$", text):
async for audio in tts.run_tts(current_text):
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio))
current_text = ""
has_joined = True
await asyncio.gather(llm_task, tts.run())
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()
@@ -56,6 +60,5 @@ if __name__ == "__main__":
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args: argparse.Namespace = parser.parse_args()
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

4
src/samples/theoretical-to-real/03-still-frame.py
View File

@@ -1,7 +1,7 @@
import argparse
import asyncio
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.open_ai_services import OpenAIImageGenService
@@ -27,7 +27,7 @@ async def main(room_url):
@transport.event_handler("on_participant_joined")
async def on_participant_joined(transport, participant):
(_, image_bytes) = await image_task
transport.output_queue.put(OutputQueueFrame(FrameType.IMAGE_FRAME, image_bytes))
transport.output_queue.put(QueueFrame(FrameType.IMAGE_FRAME, image_bytes))
await transport.run()

6
src/samples/theoretical-to-real/04-utterance-and-speech.py
View File

@@ -4,7 +4,7 @@ import re
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
async def main(room_url:str):
global transport
@@ -37,11 +37,11 @@ async def main(room_url:str):
))
async for audio_chunk in tts.run_tts("My friend the LLM is now going to tell a joke about llamas."):
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio_chunk))
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio_chunk))
llm_response = await llm_response_task
async for audio_chunk in tts.run_tts(llm_response):
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio_chunk))
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio_chunk))
# wait for the output queue to be empty, then leave the meeting

8
src/samples/theoretical-to-real/05-queued.py
View File

@@ -4,7 +4,7 @@ import asyncio
from asyncio.queues import Queue
import re
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.open_ai_services import OpenAIImageGenService
@@ -95,12 +95,12 @@ async def main(room_url):
data = await month_data_task
transport.output_queue.put(
[
OutputQueueFrame(FrameType.IMAGE_FRAME, data["image"]),
OutputQueueFrame(FrameType.AUDIO_FRAME, data["audio"][0]),
QueueFrame(FrameType.IMAGE_FRAME, data["image"]),
QueueFrame(FrameType.AUDIO_FRAME, data["audio"][0]),
]
)
for audio in data["audio"][1:]:
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio))
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()

8
src/samples/theoretical-to-real/05-sync-speech-and-text.py
View File

@@ -5,7 +5,7 @@ import asyncio
from asyncio.queues import Queue
import re
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.open_ai_services import OpenAILLMService, OpenAIImageGenService
@@ -97,12 +97,12 @@ async def main(room_url):
data = await month_data_task
transport.output_queue.put(
[
OutputQueueFrame(FrameType.IMAGE_FRAME, data["image"]),
OutputQueueFrame(FrameType.AUDIO_FRAME, data["audio"][0]),
QueueFrame(FrameType.IMAGE_FRAME, data["image"]),
QueueFrame(FrameType.AUDIO_FRAME, data["audio"][0]),
]
)
for audio in data["audio"][1:]:
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio))
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()

4
src/samples/theoretical-to-real/06-listen-and-respond.py
View File

@@ -6,7 +6,7 @@ import urllib.parse
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.output_queue import OutputQueueFrame, FrameType
from dailyai.queue_frame import QueueFrame, FrameType
async def main(room_url:str, token):
global transport
@@ -35,7 +35,7 @@ async def main(room_url:str, token):
return
async for audio_chunk in tts.run_tts("If you say something, I will respond."):
transport.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio_chunk))
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio_chunk))
@transport.event_handler("on_transcription_message")
async def on_transcription_message(transport, message) -> None: