wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4 from daily-co/service-refactor

Browse Source 
Service refactor: Generators over queues
This commit is contained in:
Moishe Lettvin
2024-01-18 11:30:59 -05:00
committed by GitHub
parent 6e8ebbd34c 0245f98eb5
commit f55333844e
14 changed files with 385 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

193
src/dailyai/services/ai_services.py
View File

@@ -2,84 +2,135 @@ import asyncio
import logging
import re
from httpx import request
from dailyai.queue_frame import QueueFrame, FrameType
from abc import abstractmethod
from typing import AsyncGenerator
from typing import AsyncGenerator, Iterable
from dataclasses import dataclass
from typing import AsyncGenerator
from collections.abc import Iterable, AsyncIterable
class AIService:
def __init__(
self,
input_queue: asyncio.Queue[QueueFrame] | None = None,
output_queue: asyncio.Queue[QueueFrame] | None = None,
):
def __init__(self):
self.logger = logging.getLogger("dailyai")
self.input_queue: asyncio.Queue[QueueFrame] | None = input_queue
self.output_queue: asyncio.Queue[QueueFrame] | None = output_queue
def stop(self):
pass
async def run(self) -> None:
if self.input_queue is None or self.output_queue is None:
raise Exception("Input and output queues must be set before using the run method.")
def allowed_input_frame_types(self) -> set[FrameType]:
return set()
while True:
frame = await self.input_queue.get()
self.logger.debug(f"{self.__class__.__name__} got frame:", frame.frame_type)
if frame.frame_type == FrameType.END_STREAM:
self.input_queue.task_done()
await self.output_queue.put(QueueFrame(FrameType.END_STREAM, None))
break
def possible_output_frame_types(self) -> set[FrameType]:
return set()
output_frame = await self.process_frame(frame)
if output_frame:
await self.output_queue.put(output_frame)
self.input_queue.task_done()
async def run_to_queue(self, queue: asyncio.Queue, frames, add_end_of_stream=False) -> None:
async for frame in self.run(frames):
await queue.put(frame)
if add_end_of_stream:
await queue.put(QueueFrame(FrameType.END_STREAM, None))
async def run(
self,
frames: Iterable[QueueFrame]
| AsyncIterable[QueueFrame]
| asyncio.Queue[QueueFrame],
requested_frame_types: set[FrameType] | None=None,
) -> AsyncGenerator[QueueFrame, None]:
if requested_frame_types and self.possible_output_frame_types().intersection(requested_frame_types) == set():
raise Exception(f"Requested frame types {requested_frame_types} are not supported by this service.")
if not requested_frame_types:
requested_frame_types = self.possible_output_frame_types()
if isinstance(frames, AsyncIterable):
async for frame in frames:
async for output_frame in self.process_frame(requested_frame_types, frame):
yield output_frame
elif isinstance(frames, Iterable):
for frame in frames:
async for output_frame in self.process_frame(requested_frame_types, frame):
yield output_frame
elif isinstance(frames, asyncio.Queue):
while True:
frame = await frames.get()
async for output_frame in self.process_frame(requested_frame_types, frame):
yield output_frame
if frame.frame_type == FrameType.END_STREAM:
break
else:
raise Exception("Frames must be an iterable or async iterable")
@abstractmethod
async def process_frame(self, frame) -> QueueFrame | None:
pass
async def process_frame(self, requested_frame_types:set[FrameType], frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]:
# Yield something so the linter can deduce what should happen here.
yield QueueFrame(FrameType.END_STREAM, None)
class SentenceAggregator(AIService):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.current_sentence = ""
def allowed_input_frame_types(self) -> set[FrameType]:
return set([FrameType.TEXT_CHUNK, FrameType.SENTENCE])
def possible_output_frame_types(self) -> set[FrameType]:
return set([FrameType.SENTENCE])
async def process_frame(self, requested_frame_types: set[FrameType], frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if not FrameType.SENTENCE in requested_frame_types:
return
if frame.frame_type == FrameType.TEXT_CHUNK:
if type(frame.frame_data) != str:
raise Exception(
"Sentence aggregator requires a string for the data field"
)
self.current_sentence += frame.frame_data
if self.current_sentence.endswith((".", "?", "!")):
sentence = self.current_sentence
self.current_sentence = ""
yield QueueFrame(FrameType.SENTENCE, sentence)
elif frame.frame_type == FrameType.END_STREAM:
if self.current_sentence:
yield QueueFrame(FrameType.SENTENCE, self.current_sentence)
elif frame.frame_type == FrameType.SENTENCE:
yield frame
class LLMService(AIService):
# Generate a set of responses to a prompt. Yields a list of responses.
def allowed_input_frame_types(self) -> set[FrameType]:
return set([FrameType.LLM_MESSAGE, FrameType.SENTENCE, FrameType.TRANSCRIPTION])
def allowed_output_frame_types(self) -> set[FrameType]:
return set([FrameType.SENTENCE, FrameType.TEXT_CHUNK])
@abstractmethod
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
# Adding a yield here lets the linter know what this method actually does
yield ""
# Generate a responses to a prompt. Returns the response
@abstractmethod
async def run_llm(
self, messages
) -> str or None:
async def run_llm(self, messages) -> str:
pass
async def run_llm_async_sentences(self, messages) -> AsyncGenerator[str, None]:
current_text = ""
async for text in self.run_llm_async(messages):
current_text += text
if re.match(r"^.*[.!?]$", text):
yield current_text
current_text = ""
if current_text:
yield current_text
async def process_frame(self, frame:QueueFrame) -> QueueFrame | None:
if not self.output_queue:
raise Exception("Output queue must be set before using the run method.")
async def process_frame(self, requested_frame_types: set[FrameType], frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if frame.frame_type == FrameType.LLM_MESSAGE:
if type(frame.frame_data) != list:
raise Exception("LLM service requires a dict for the data field")
messages: list[dict[str, str]] = frame.frame_data
async for message in self.run_llm_async_sentences(messages):
await self.output_queue.put(QueueFrame(FrameType.SENTENCE, message))
if FrameType.SENTENCE in requested_frame_types:
yield QueueFrame(FrameType.SENTENCE, await self.run_llm(messages))
else:
async for text_chunk in self.run_llm_async(messages):
yield QueueFrame(FrameType.TEXT_CHUNK, text_chunk)
# TODO: handle other frame types! Need to aggregate into messages
class TTSService(AIService):
@@ -87,6 +138,12 @@ class TTSService(AIService):
def get_mic_sample_rate(self):
return 16000
def allowed_input_frame_types(self) -> set[FrameType]:
return set([FrameType.SENTENCE, FrameType.TRANSCRIPTION, FrameType.TEXT_CHUNK])
def possible_output_frame_types(self) -> set[FrameType]:
return set([FrameType.AUDIO])
# Converts the sentence to audio. Yields a list of audio frames that can
# be sent to the microphone device
@abstractmethod
@@ -94,25 +151,47 @@ class TTSService(AIService):
# yield empty bytes here, so linting can infer what this method does
yield bytes()
async def process_frame(self, frame:QueueFrame) -> QueueFrame | None:
if not self.output_queue:
raise Exception("Output queue must be set before using the run method.")
async def process_frame(self, requested_frame_types: set[FrameType], frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if not FrameType.AUDIO in requested_frame_types:
return
if frame.frame_type == FrameType.SENTENCE:
if type(frame.frame_data) != str:
raise Exception("TTS service requires a string for the data field")
if type(frame.frame_data) != str:
raise Exception("TTS service requires a string for the data field")
text = frame.frame_data
async for audio in self.run_tts(text):
await self.output_queue.put(QueueFrame(FrameType.AUDIO, audio))
async for audio_chunk in self.run_tts(frame.frame_data):
yield QueueFrame(FrameType.AUDIO, audio_chunk)
# Convenience function to send the audio for a sentence to the given queue
async def say(self, sentence, queue: asyncio.Queue):
await self.run_to_queue(queue, [QueueFrame(FrameType.SENTENCE, sentence)])
class ImageGenService(AIService):
def __init__(self, image_size, **kwargs):
super().__init__(**kwargs)
self.image_size = image_size
def allowed_input_frame_types(self) -> set[FrameType]:
return set([FrameType.SENTENCE, FrameType.TRANSCRIPTION, FrameType.TEXT_CHUNK, FrameType.IMAGE_DESCRIPTION])
def possible_output_frame_types(self) -> set[FrameType]:
return set([FrameType.IMAGE])
# Renders the image. Returns an Image object.
@abstractmethod
async def run_image_gen(self, sentence, size) -> tuple[str, bytes]:
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
pass
async def process_frame(self, requested_frame_types: set[FrameType], frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if not FrameType.IMAGE in requested_frame_types:
return
if type(frame.frame_data) != str:
raise Exception("Image service requires a string for the data field")
(_, image_data) = await self.run_image_gen(frame.frame_data)
yield QueueFrame(FrameType.IMAGE, image_data)
@dataclass
class AIServiceConfig:

25
src/dailyai/services/azure_ai_services.py
View File

@@ -16,8 +16,8 @@ from PIL import Image
from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason
class AzureTTSService(TTSService):
def __init__(self, input_queue=None, output_queue=None, speech_key=None, speech_region=None):
super().__init__(input_queue, output_queue)
def __init__(self, speech_key=None, speech_region=None):
super().__init__()
speech_key = speech_key or os.getenv("AZURE_SPEECH_SERVICE_KEY")
speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION")
@@ -35,7 +35,10 @@ class AzureTTSService(TTSService):
"<prosody rate='1.05'>" \
f"{sentence}" \
"</prosody></mstts:express-as></voice></speak> "
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
try:
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
except Exception as e:
self.logger.error("Error in azure tts", e)
self.logger.info("Got azure tts result")
if result.reason == ResultReason.SynthesizingAudioCompleted:
self.logger.info("Returning result")
@@ -48,8 +51,8 @@ class AzureTTSService(TTSService):
self.logger.info("Error details: {}".format(cancellation_details.error_details))
class AzureLLMService(LLMService):
def __init__(self, input_queue=None, output_queue=None, api_key=None, azure_endpoint=None, api_version=None, model=None):
super().__init__(input_queue, output_queue)
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None):
super().__init__()
api_key = api_key or os.getenv("AZURE_CHATGPT_KEY")
azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT")
@@ -92,14 +95,14 @@ class AzureLLMService(LLMService):
class AzureImageGenServiceREST(ImageGenService):
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None):
super().__init__()
def __init__(self, image_size:str, api_key=None, azure_endpoint=None, api_version=None, model=None):
super().__init__(image_size=image_size)
self.api_key = api_key or os.getenv("AZURE_DALLE_KEY")
self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
self.api_version = api_version or "2023-06-01-preview"
self.model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID")
async def run_image_gen(self, sentence, size) -> tuple[str, bytes]:
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
# TODO hoist the session to app-level
async with aiohttp.ClientSession() as session:
url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}"
@@ -107,7 +110,7 @@ class AzureImageGenServiceREST(ImageGenService):
body = {
# Enter your prompt text here
"prompt": sentence,
"size": size,
"size": self.image_size,
"n": 1,
}
async with session.post(url, headers=headers, json=body) as submission:
@@ -153,14 +156,14 @@ class AzureImageGenService(ImageGenService):
api_version=api_version,
)
async def run_image_gen(self, sentence, size) -> tuple[str, bytes]:
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
self.logger.info("Generating azure image", sentence)
image = self.client.images.generate(
model=self.model,
prompt=sentence,
n=1,
size=size,
size=self.image_size,
)
url = image["data"][0]["url"]

11
src/dailyai/services/daily_transport_service.py
View File

@@ -200,12 +200,19 @@ class DailyTransportService(EventHandler):
async def marshal_frames(self):
while True:
frame = await self.send_queue.get()
frame: QueueFrame | list = await self.send_queue.get()
self.threadsafe_send_queue.put(frame)
self.send_queue.task_done()
if frame.frame_type == FrameType.END_STREAM:
if type(frame) == QueueFrame and frame.frame_type == FrameType.END_STREAM:
break
def wait_for_send_queue_to_empty(self):
self.threadsafe_send_queue.join()
def stop_when_done(self):
self.wait_for_send_queue_to_empty()
self.stop()
async def run(self) -> None:
self.configure_daily()

4
src/dailyai/services/elevenlabs_ai_service.py
View File

@@ -9,8 +9,8 @@ from dailyai.services.ai_services import TTSService
class ElevenLabsTTSService(TTSService):
def __init__(self, input_queue=None, output_queue=None, api_key=None, voice_id=None):
super().__init__(input_queue, output_queue)
def __init__(self, api_key=None, voice_id=None):
super().__init__()
self.api_key = api_key or os.getenv("ELEVENLABS_API_KEY")
self.voice_id = voice_id or os.getenv("ELEVENLABS_VOICE_ID")

12
src/dailyai/services/fal_ai_services.py
View File

@@ -9,12 +9,10 @@ from PIL import Image
from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
class FalImageGenService(ImageGenService):
def __init__(self):
super().__init__()
def __init__(self, image_size):
super().__init__(image_size)
async def run_image_gen(self, sentence, size) -> tuple[str, bytes]:
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
def get_image_url(sentence, size):
print("starting fal submit...")
handler = fal.apps.submit(
@@ -37,7 +35,7 @@ class FalImageGenService(ImageGenService):
return image_url
print(f"fetching image url...")
image_url = await asyncio.to_thread(get_image_url, sentence, size)
image_url = await asyncio.to_thread(get_image_url, sentence, self.image_size)
print(f"got image url, downloading image...")
# Load the image from the url
async with aiohttp.ClientSession() as session:
@@ -48,4 +46,4 @@ class FalImageGenService(ImageGenService):
image = Image.open(image_stream)
return (image_url, image.tobytes())
# return (image_url, dalle_im.tobytes())
# return (image_url, dalle_im.tobytes())

8
src/dailyai/services/open_ai_services.py
View File

@@ -50,20 +50,20 @@ class OpenAILLMService(LLMService):
return None
class OpenAIImageGenService(ImageGenService):
def __init__(self, api_key=None, model=None):
super().__init__()
def __init__(self, image_size:str, api_key=None, model=None):
super().__init__(image_size=image_size)
api_key = api_key or os.getenv("OPEN_AI_KEY")
self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3"
self.client = AsyncOpenAI(api_key=api_key)
async def run_image_gen(self, sentence, size) -> tuple[str, bytes]:
async def run_image_gen(self, sentence) -> tuple[str, bytes]:
self.logger.info("Generating OpenAI image", sentence)
image = await self.client.images.generate(
prompt=sentence,
model=self.model,
n=1,
size=size
size=self.image_size
)
image_url = image.data[0].url
if not image_url:

1
src/dailyai/services/to_be_updated/huggingface_ai_service.py
View File

@@ -13,7 +13,6 @@ class HuggingFaceAIService(AIService):
# available models at https://huggingface.co/Helsinki-NLP (**not all models use 2-character language codes**)
def run_text_translation(self, sentence, source_language, target_language):
translator = pipeline(f"translation", model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
print(translator(sentence))
return translator(sentence)[0]["translation_text"]

129
src/dailyai/tests/test_ai_services.py Normal file
View File

@@ -0,0 +1,129 @@
from re import A
import unittest
from typing import AsyncGenerator, Generator
from dailyai.services.ai_services import AIService, SentenceAggregator
from dailyai.queue_frame import QueueFrame, FrameType
class SimpleAIService(AIService):
def allowed_input_frame_types(self) -> set[FrameType]:
return set([FrameType.TEXT_CHUNK])
def possible_output_frame_types(self) -> set[FrameType]:
return set([FrameType.TEXT_CHUNK])
async def process_frame(self, requested_frame_types: set[FrameType], frame: QueueFrame) -> QueueFrame | None:
return frame
class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
async def test_async_input(self):
service = SimpleAIService()
input_frames = [
QueueFrame(FrameType.TEXT_CHUNK, "hello"),
QueueFrame(FrameType.END_STREAM, None),
]
async def iterate_frames() -> AsyncGenerator[QueueFrame, None]:
for frame in input_frames:
yield frame
output_frames = []
async for frame in service.run(set([FrameType.TEXT_CHUNK]), iterate_frames()):
output_frames.append(frame)
self.assertEqual(input_frames, output_frames)
async def test_nonasync_input(self):
service = SimpleAIService()
input_frames = [
QueueFrame(FrameType.TEXT_CHUNK, "hello"),
QueueFrame(FrameType.END_STREAM, None),
]
def iterate_frames() -> Generator[QueueFrame, None, None]:
for frame in input_frames:
yield frame
output_frames = []
async for frame in service.run(set([FrameType.TEXT_CHUNK]), iterate_frames()):
output_frames.append(frame)
self.assertEqual(input_frames, output_frames)
class TestSentenceAggregator(unittest.IsolatedAsyncioTestCase):
async def test_clause(self) -> None:
input_frames = [
QueueFrame(FrameType.TEXT_CHUNK, "hello"),
QueueFrame(FrameType.END_STREAM, None),
]
service = SentenceAggregator()
output_frames = []
async for frame in service.run(set([FrameType.SENTENCE]), input_frames):
output_frames.append(frame)
self.assertEqual(1, len(output_frames))
self.assertEqual(QueueFrame(FrameType.SENTENCE, "hello"), output_frames[0])
async def test_sentence(self) -> None:
input_frames = [
QueueFrame(FrameType.TEXT_CHUNK, "hello, "),
QueueFrame(FrameType.TEXT_CHUNK, "world."),
QueueFrame(FrameType.END_STREAM, None),
]
service = SentenceAggregator()
output_frames = []
async for frame in service.run(set([FrameType.SENTENCE]), input_frames):
output_frames.append(frame)
self.assertEqual(1, len(output_frames))
self.assertEqual(QueueFrame(FrameType.SENTENCE, "hello, world."), output_frames[0])
async def test_sentence_and_clause(self) -> None:
input_frames = [
QueueFrame(FrameType.TEXT_CHUNK, "hello, "),
QueueFrame(FrameType.TEXT_CHUNK, "world."),
QueueFrame(FrameType.TEXT_CHUNK, " How are"),
QueueFrame(FrameType.END_STREAM, None),
]
service = SentenceAggregator()
output_frames = []
async for frame in service.run(set([FrameType.SENTENCE]), input_frames):
output_frames.append(frame)
self.assertEqual(2, len(output_frames))
self.assertEqual(
QueueFrame(FrameType.SENTENCE, "hello, world."), output_frames[0]
)
self.assertEqual(
QueueFrame(FrameType.SENTENCE, " How are"), output_frames[1]
)
async def test_two_sentences(self) -> None:
input_frames = [
QueueFrame(FrameType.TEXT_CHUNK, "hello, "),
QueueFrame(FrameType.TEXT_CHUNK, "world."),
QueueFrame(FrameType.TEXT_CHUNK, " How are"),
QueueFrame(FrameType.TEXT_CHUNK, " you doing?"),
QueueFrame(FrameType.END_STREAM, None),
]
service = SentenceAggregator()
output_frames = []
async for frame in service.run(set([FrameType.SENTENCE]), input_frames):
output_frames.append(frame)
self.assertEqual(2, len(output_frames))
self.assertEqual(
QueueFrame(FrameType.SENTENCE, "hello, world."), output_frames[0]
)
self.assertEqual(QueueFrame(FrameType.SENTENCE, " How are you doing?"), output_frames[1])
if __name__ == "__main__":
unittest.main()

9
src/dailyai/tests/test_asyncprocessor.py
View File

@@ -18,7 +18,7 @@ from dailyai.services.ai_services import (
LLMService,
TTSService,
)
"""
class MockTTSService(TTSService):
def run_tts(self, sentence):
for word in sentence.split(' '):
@@ -73,7 +73,7 @@ class TestResponse(unittest.TestCase):
while expected_words:
actual_word:QueueFrame = output_queue.get()
word = expected_words.pop(0)
self.assertEqual(actual_word.frame_type, FrameType.AUDIO)
self.assertEqual(actual_word.frame_type, FrameType.AUDIO_FRAME)
self.assertEqual(actual_word.frame_data, bytes(word, "utf-8"))
output_queue.task_done()
@@ -128,10 +128,10 @@ class TestResponse(unittest.TestCase):
while expected_words and not stop_processing_output_queue.is_set():
try:
actual_word:QueueFrame = output_queue.get_nowait()
if actual_word.frame_type == FrameType.AUDIO:
if actual_word.frame_type == FrameType.AUDIO_FRAME:
time.sleep(0.1)
word = expected_words.pop(0)
self.assertEqual(actual_word.frame_type, FrameType.AUDIO)
self.assertEqual(actual_word.frame_type, FrameType.AUDIO_FRAME)
self.assertEqual(actual_word.frame_data, bytes(word, "utf-8"))
output_queue.task_done()
except Empty:
@@ -177,3 +177,4 @@ class TestResponse(unittest.TestCase):
if __name__ == '__main__':
unittest.main()
"""

18
src/samples/theoretical-to-real/01-say-one-thing.py
View File

@@ -5,6 +5,7 @@ from typing import AsyncGenerator
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureTTSService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
async def main(room_url):
# create a transport service object using environment variables for
@@ -23,13 +24,7 @@ async def main(room_url):
meeting_duration_minutes,
)
transport.mic_enabled = True
# similarly, create a tts service
tts = AzureTTSService()
# Get the generator for the audio. This will start running in the background,
# and when we ask the generator for its items, we'll get what it's generated.
audio_generator: AsyncGenerator[bytes, None] = tts.run_tts("hello world")
tts = ElevenLabsTTSService(voice_id="ErXwobaYiN019PkySvjV")
# Register an event handler so we can play the audio when the participant joins.
@transport.event_handler("on_participant_joined")
@@ -37,12 +32,13 @@ async def main(room_url):
if participant["info"]["isLocal"]:
return
async for audio in audio_generator:
transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio))
await tts.say(
"Hello there, " + participant["info"]["userName"] + "!",
transport.send_queue,
)
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()
transport.stop()
transport.stop_when_done()
await transport.run()

28
src/samples/theoretical-to-real/02-llm-say-one-thing.py
View File

@@ -4,6 +4,7 @@ from typing import AsyncGenerator
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.ai_services import SentenceAggregator
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
@@ -17,30 +18,27 @@ async def main(room_url):
)
transport.mic_enabled = True
text_to_llm_queue = asyncio.Queue()
llm_to_tts_queue = asyncio.Queue()
tts = ElevenLabsTTSService(
llm_to_tts_queue, transport.get_async_send_queue(), voice_id="29vD33N1CtxCmqQRPOHJ"
)
llm = AzureLLMService(text_to_llm_queue, llm_to_tts_queue)
tts = ElevenLabsTTSService(voice_id="29vD33N1CtxCmqQRPOHJ")
llm = AzureLLMService()
messages = [{
"role": "system",
"content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world."
}]
await text_to_llm_queue.put(QueueFrame(FrameType.LLM_MESSAGE, messages))
await text_to_llm_queue.put(QueueFrame(FrameType.END_STREAM, None))
llm_task = asyncio.create_task(llm.run())
tts_task = asyncio.create_task(
tts.run_to_queue(
transport.send_queue,
SentenceAggregator().run(
llm.run([QueueFrame(FrameType.LLM_MESSAGE, messages)])
)
)
)
@transport.event_handler("on_first_other_participant_joined")
async def on_first_other_participant_joined(transport):
await asyncio.gather(llm_task, tts.run())
await tts_task
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()
transport.stop()
transport.stop_when_done()
await transport.run()

11
src/samples/theoretical-to-real/03-still-frame.py
View File

@@ -21,13 +21,14 @@ async def main(room_url):
transport.camera_width = 1024
transport.camera_height = 1024
imagegen = OpenAIImageGenService()
image_task = asyncio.create_task(imagegen.run_image_gen("a cat in the style of picasso", "1024x1024"))
imagegen = OpenAIImageGenService(image_size="1024x1024")
image_task = asyncio.create_task(
imagegen.run_to_queue(transport.send_queue, [QueueFrame(FrameType.IMAGE_DESCRIPTION, "a cat in the style of picasso")])
)
@transport.event_handler("on_participant_joined")
async def on_participant_joined(transport, participant):
(_, image_bytes) = await image_task
transport.output_queue.put(QueueFrame(FrameType.IMAGE, image_bytes))
await image_task
await transport.run()
@@ -38,6 +39,6 @@ if __name__ == "__main__":
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args: argparse.Namespace = parser.parse_args()
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

55
src/samples/theoretical-to-real/04-utterance-and-speech.py
View File

@@ -2,9 +2,11 @@ import argparse
import asyncio
import re
from dailyai.services.ai_services import SentenceAggregator
from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
async def main(room_url:str):
global transport
@@ -22,35 +24,46 @@ async def main(room_url:str):
transport.camera_enabled = False
llm = AzureLLMService()
tts = AzureTTSService()
azure_tts = AzureTTSService()
elevenlabs_tts = ElevenLabsTTSService(voice_id="ErXwobaYiN019PkySvjV")
messages = [{"role": "system", "content": "tell the user a joke about llamas"}]
# Start a task to run the LLM to create a joke, and convert the LLM output to audio frames. This task
# will run in parallel with generating and speaking the audio for static text, so there's no delay to
# speak the LLM response.
buffer_queue = asyncio.Queue()
llm_response_task = asyncio.create_task(
elevenlabs_tts.run_to_queue(
buffer_queue,
SentenceAggregator().run(
llm.run([QueueFrame(FrameType.LLM_MESSAGE, messages)])
),
True,
)
)
@transport.event_handler("on_participant_joined")
async def on_joined(transport, participant):
if participant["id"] == transport.my_participant_id:
return
# queue two pieces of speech: one specified as a text literal,
# and one generated by an llm. We'll kick off the llm first, and let
# it generate a response while we're speaking the literal string.
#
# Note that in this case, we don't use `run_llm_async` because we're
# taking advantage of the time spent speaking the first phrase to generate
# the entire LLM response, and this happens asynchronously in a task.
llm_response_task = asyncio.create_task(llm.run_llm(
[{"role": "system", "content": "tell the user a joke about llamas"}]
))
await azure_tts.run_to_queue(
transport.send_queue,
[QueueFrame(FrameType.SENTENCE, "My friend the LLM is now going to tell a joke about llamas.")]
)
async for audio_chunk in tts.run_tts("My friend the LLM is now going to tell a joke about llamas."):
transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio_chunk))
async def buffer_to_send_queue():
while True:
frame = await buffer_queue.get()
await transport.send_queue.put(frame)
buffer_queue.task_done()
if frame.frame_type == FrameType.END_STREAM:
break
llm_response = await llm_response_task
async for audio_chunk in tts.run_tts(llm_response):
transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio_chunk))
await asyncio.gather(llm_response_task, buffer_to_send_queue())
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()
transport.stop()
transport.stop_when_done()
await transport.run()
@@ -61,6 +74,6 @@ if __name__ == "__main__":
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
)
args: argparse.Namespace = parser.parse_args()
args, unknown = parser.parse_known_args()
asyncio.run(main(args.url))

40
src/samples/theoretical-to-real/05-sync-speech-and-text.py
View File

@@ -5,6 +5,7 @@ from asyncio.queues import Queue
import re
from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.ai_services import SentenceAggregator
from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.open_ai_services import OpenAIImageGenService
@@ -26,13 +27,12 @@ async def main(room_url):
transport.camera_height = 1024
llm = AzureLLMService()
#tts = ElevenLabsTTSService(voice_id="ErXwobaYiN019PkySvjV")
tts = ElevenLabsTTSService()
dalle = FalImageGenService()
# dalle = OpenAIImageGenService()
dalle = FalImageGenService(image_size="1024x1024")
tts = ElevenLabsTTSService(voice_id="ErXwobaYiN019PkySvjV")
# dalle = OpenAIImageGenService(image_size="1024x1024")
# Get a complete audio chunk from the given text. Splitting this into its own
# coroutine lets us ensure proper ordering of the audio chunks on the output queue.
# coroutine lets us ensure proper ordering of the audio chunks on the send queue.
async def get_all_audio(text):
all_audio = bytearray()
async for audio in tts.run_tts(text):
@@ -44,14 +44,18 @@ async def main(room_url):
image_text = ""
tts_tasks = []
first_sentence = True
async for sentence in llm.run_llm_async_sentences(
[
{
"role": "system",
"content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please."
}
]
):
messages = [
{
"role": "system",
"content": f"Describe a nature photograph suitable for use in a calendar, for the month of {month}. Include only the image description with no preamble. Limit the description to one sentence, please.",
}
]
async for frame in SentenceAggregator().run(llm.run([QueueFrame(FrameType.LLM_MESSAGE, messages)])):
if type(frame.frame_data) != str:
raise Exception("LLM service requires a string for the data field")
sentence: str = frame.frame_data
image_text += sentence
if first_sentence:
@@ -61,7 +65,7 @@ async def main(room_url):
tts_tasks.append(get_all_audio(sentence))
tts_tasks.insert(0, dalle.run_image_gen(image_text, "1024x1024"))
tts_tasks.insert(0, dalle.run_image_gen(image_text))
print(f"waiting for tasks to finish for {month}")
data = await asyncio.gather(
@@ -101,19 +105,17 @@ async def main(room_url):
# likely no delay between months, but the months won't display in order.
for month_data_task in asyncio.as_completed(month_tasks):
data = await month_data_task
print(f"got data, queueing frames...")
transport.output_queue.put(
await transport.send_queue.put(
[
QueueFrame(FrameType.IMAGE, data["image"]),
QueueFrame(FrameType.AUDIO, data["audio"][0]),
]
)
for audio in data["audio"][1:]:
transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio))
await transport.send_queue.put(QueueFrame(FrameType.AUDIO, audio))
# wait for the output queue to be empty, then leave the meeting
transport.output_queue.join()
transport.stop()
transport.stop_when_done()
month_tasks = [asyncio.create_task(get_month_data(month)) for month in months]