@@ -5,6 +5,7 @@ from typing import AsyncGenerator, Awaitable, Callable
|
|||||||
from dailyai.queue_aggregators import LLMContextAggregator
|
from dailyai.queue_aggregators import LLMContextAggregator
|
||||||
from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TranscriptionQueueFrame
|
from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TranscriptionQueueFrame
|
||||||
|
|
||||||
|
|
||||||
class InterruptibleConversationWrapper:
|
class InterruptibleConversationWrapper:
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -14,7 +15,7 @@ class InterruptibleConversationWrapper:
|
|||||||
[str, LLMContextAggregator, LLMContextAggregator], Awaitable[None]
|
[str, LLMContextAggregator, LLMContextAggregator], Awaitable[None]
|
||||||
],
|
],
|
||||||
interrupt: Callable[[], None],
|
interrupt: Callable[[], None],
|
||||||
my_participant_id: str|None,
|
my_participant_id: str | None,
|
||||||
llm_messages: list[dict[str, str]],
|
llm_messages: list[dict[str, str]],
|
||||||
llm_context_aggregator_in=LLMContextAggregator,
|
llm_context_aggregator_in=LLMContextAggregator,
|
||||||
llm_context_aggregator_out=LLMContextAggregator,
|
llm_context_aggregator_out=LLMContextAggregator,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from dailyai.services.ai_services import AIService
|
|||||||
|
|
||||||
from typing import AsyncGenerator, List
|
from typing import AsyncGenerator, List
|
||||||
|
|
||||||
|
|
||||||
class QueueTee:
|
class QueueTee:
|
||||||
async def run_to_queue_and_generate(
|
async def run_to_queue_and_generate(
|
||||||
self,
|
self,
|
||||||
@@ -24,15 +25,21 @@ class QueueTee:
|
|||||||
for queue in output_queues:
|
for queue in output_queues:
|
||||||
await queue.put(frame)
|
await queue.put(frame)
|
||||||
|
|
||||||
|
|
||||||
class LLMContextAggregator(AIService):
|
class LLMContextAggregator(AIService):
|
||||||
def __init__(self, messages: list[dict], role:str, bot_participant_id=None, complete_sentences=True):
|
def __init__(
|
||||||
|
self,
|
||||||
|
messages: list[dict],
|
||||||
|
role: str,
|
||||||
|
bot_participant_id=None,
|
||||||
|
complete_sentences=True):
|
||||||
self.messages = messages
|
self.messages = messages
|
||||||
self.bot_participant_id = bot_participant_id
|
self.bot_participant_id = bot_participant_id
|
||||||
self.role = role
|
self.role = role
|
||||||
self.sentence = ""
|
self.sentence = ""
|
||||||
self.complete_sentences = complete_sentences
|
self.complete_sentences = complete_sentences
|
||||||
|
|
||||||
async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
||||||
# TODO: split up transcription by participant
|
# TODO: split up transcription by participant
|
||||||
if isinstance(frame, TextQueueFrame):
|
if isinstance(frame, TextQueueFrame):
|
||||||
if self.complete_sentences:
|
if self.complete_sentences:
|
||||||
|
|||||||
@@ -2,39 +2,49 @@ from enum import Enum
|
|||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
class QueueFrame:
|
class QueueFrame:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class ControlQueueFrame(QueueFrame):
|
class ControlQueueFrame(QueueFrame):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class StartStreamQueueFrame(ControlQueueFrame):
|
class StartStreamQueueFrame(ControlQueueFrame):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class EndStreamQueueFrame(ControlQueueFrame):
|
class EndStreamQueueFrame(ControlQueueFrame):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class AudioQueueFrame(QueueFrame):
|
class AudioQueueFrame(QueueFrame):
|
||||||
data: bytes
|
data: bytes
|
||||||
|
|
||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class ImageQueueFrame(QueueFrame):
|
class ImageQueueFrame(QueueFrame):
|
||||||
url: str | None
|
url: str | None
|
||||||
image: bytes
|
image: bytes
|
||||||
|
|
||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class TextQueueFrame(QueueFrame):
|
class TextQueueFrame(QueueFrame):
|
||||||
text: str
|
text: str
|
||||||
|
|
||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class TranscriptionQueueFrame(TextQueueFrame):
|
class TranscriptionQueueFrame(TextQueueFrame):
|
||||||
participantId: str
|
participantId: str
|
||||||
timestamp: str
|
timestamp: str
|
||||||
|
|
||||||
|
|
||||||
@dataclass()
|
@dataclass()
|
||||||
class LLMMessagesQueueFrame(QueueFrame):
|
class LLMMessagesQueueFrame(QueueFrame):
|
||||||
messages: list[dict[str,str]] # TODO: define this more concretely!
|
messages: list[dict[str, str]] # TODO: define this more concretely!
|
||||||
|
|
||||||
|
|
||||||
class AppMessageQueueFrame(QueueFrame):
|
class AppMessageQueueFrame(QueueFrame):
|
||||||
message: Any
|
message: Any
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ class AIService:
|
|||||||
raise e
|
raise e
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
||||||
if isinstance(frame, ControlQueueFrame):
|
if isinstance(frame, ControlQueueFrame):
|
||||||
yield frame
|
yield frame
|
||||||
|
|
||||||
@@ -75,6 +75,7 @@ class AIService:
|
|||||||
if False:
|
if False:
|
||||||
yield QueueFrame()
|
yield QueueFrame()
|
||||||
|
|
||||||
|
|
||||||
class LLMService(AIService):
|
class LLMService(AIService):
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
|
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
|
||||||
@@ -146,7 +147,7 @@ class ImageGenService(AIService):
|
|||||||
|
|
||||||
# Renders the image. Returns an Image object.
|
# Renders the image. Returns an Image object.
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def run_image_gen(self, sentence:str) -> tuple[str, bytes]:
|
async def run_image_gen(self, sentence: str) -> tuple[str, bytes]:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
||||||
@@ -157,15 +158,16 @@ class ImageGenService(AIService):
|
|||||||
(url, image_data) = await self.run_image_gen(frame.text)
|
(url, image_data) = await self.run_image_gen(frame.text)
|
||||||
yield ImageQueueFrame(url, image_data)
|
yield ImageQueueFrame(url, image_data)
|
||||||
|
|
||||||
|
|
||||||
class STTService(AIService):
|
class STTService(AIService):
|
||||||
"""STTService is a base class for speech-to-text services."""
|
"""STTService is a base class for speech-to-text services."""
|
||||||
|
|
||||||
_frame_rate: int
|
_frame_rate: int
|
||||||
|
|
||||||
def __init__(self, frame_rate: int = 16000, **kwargs):
|
def __init__(self, frame_rate: int = 16000, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self._frame_rate = frame_rate
|
self._frame_rate = frame_rate
|
||||||
|
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
async def run_stt(self, audio: BinaryIO) -> str:
|
async def run_stt(self, audio: BinaryIO) -> str:
|
||||||
"""Returns transcript as a string"""
|
"""Returns transcript as a string"""
|
||||||
@@ -188,6 +190,7 @@ class STTService(AIService):
|
|||||||
text = await self.run_stt(content)
|
text = await self.run_stt(content)
|
||||||
yield TextQueueFrame(text)
|
yield TextQueueFrame(text)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class AIServiceConfig:
|
class AIServiceConfig:
|
||||||
tts: TTSService
|
tts: TTSService
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from PIL import Image
|
|||||||
# See .env.example for Azure configuration needed
|
# See .env.example for Azure configuration needed
|
||||||
from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason
|
from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason
|
||||||
|
|
||||||
|
|
||||||
class AzureTTSService(TTSService):
|
class AzureTTSService(TTSService):
|
||||||
def __init__(self, speech_key=None, speech_region=None):
|
def __init__(self, speech_key=None, speech_region=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -23,18 +24,19 @@ class AzureTTSService(TTSService):
|
|||||||
speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION")
|
speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION")
|
||||||
|
|
||||||
self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region)
|
self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region)
|
||||||
self.speech_synthesizer = SpeechSynthesizer(speech_config=self.speech_config, audio_config=None)
|
self.speech_synthesizer = SpeechSynthesizer(
|
||||||
|
speech_config=self.speech_config, audio_config=None)
|
||||||
|
|
||||||
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
|
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
|
||||||
self.logger.info("Running azure tts")
|
self.logger.info("Running azure tts")
|
||||||
ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
|
ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
|
||||||
"xmlns:mstts='http://www.w3.org/2001/mstts'>" \
|
"xmlns:mstts='http://www.w3.org/2001/mstts'>" \
|
||||||
"<voice name='en-US-SaraNeural'>" \
|
"<voice name='en-US-SaraNeural'>" \
|
||||||
"<mstts:silence type='Sentenceboundary' value='20ms' />" \
|
"<mstts:silence type='Sentenceboundary' value='20ms' />" \
|
||||||
"<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>" \
|
"<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>" \
|
||||||
"<prosody rate='1.05'>" \
|
"<prosody rate='1.05'>" \
|
||||||
f"{sentence}" \
|
f"{sentence}" \
|
||||||
"</prosody></mstts:express-as></voice></speak> "
|
"</prosody></mstts:express-as></voice></speak> "
|
||||||
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
|
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
|
||||||
self.logger.info("Got azure tts result")
|
self.logger.info("Got azure tts result")
|
||||||
if result.reason == ResultReason.SynthesizingAudioCompleted:
|
if result.reason == ResultReason.SynthesizingAudioCompleted:
|
||||||
@@ -47,6 +49,7 @@ class AzureTTSService(TTSService):
|
|||||||
if cancellation_details.reason == CancellationReason.Error:
|
if cancellation_details.reason == CancellationReason.Error:
|
||||||
self.logger.info("Error details: {}".format(cancellation_details.error_details))
|
self.logger.info("Error details: {}".format(cancellation_details.error_details))
|
||||||
|
|
||||||
|
|
||||||
class AzureLLMService(LLMService):
|
class AzureLLMService(LLMService):
|
||||||
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None):
|
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -54,11 +57,13 @@ class AzureLLMService(LLMService):
|
|||||||
|
|
||||||
azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT")
|
azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT")
|
||||||
if not azure_endpoint:
|
if not azure_endpoint:
|
||||||
raise Exception("No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor")
|
raise Exception(
|
||||||
|
"No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor")
|
||||||
|
|
||||||
model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID")
|
model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID")
|
||||||
if not model:
|
if not model:
|
||||||
raise Exception("No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor")
|
raise Exception(
|
||||||
|
"No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor")
|
||||||
self.model: str = model
|
self.model: str = model
|
||||||
|
|
||||||
api_version = api_version or "2023-12-01-preview"
|
api_version = api_version or "2023-12-01-preview"
|
||||||
@@ -90,9 +95,16 @@ class AzureLLMService(LLMService):
|
|||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
class AzureImageGenServiceREST(ImageGenService):
|
class AzureImageGenServiceREST(ImageGenService):
|
||||||
|
|
||||||
def __init__(self, image_size:str, api_key=None, azure_endpoint=None, api_version=None, model=None):
|
def __init__(
|
||||||
|
self,
|
||||||
|
image_size: str,
|
||||||
|
api_key=None,
|
||||||
|
azure_endpoint=None,
|
||||||
|
api_version=None,
|
||||||
|
model=None):
|
||||||
super().__init__(image_size=image_size)
|
super().__init__(image_size=image_size)
|
||||||
self.api_key = api_key or os.getenv("AZURE_DALLE_KEY")
|
self.api_key = api_key or os.getenv("AZURE_DALLE_KEY")
|
||||||
self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
|
self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
|
||||||
@@ -103,7 +115,7 @@ class AzureImageGenServiceREST(ImageGenService):
|
|||||||
# TODO hoist the session to app-level
|
# TODO hoist the session to app-level
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}"
|
url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}"
|
||||||
headers= { "api-key": self.api_key, "Content-Type": "application/json" }
|
headers = {"api-key": self.api_key, "Content-Type": "application/json"}
|
||||||
body = {
|
body = {
|
||||||
# Enter your prompt text here
|
# Enter your prompt text here
|
||||||
"prompt": sentence,
|
"prompt": sentence,
|
||||||
|
|||||||
@@ -30,6 +30,7 @@ from daily import (
|
|||||||
VirtualSpeakerDevice,
|
VirtualSpeakerDevice,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
class DailyTransportService(EventHandler):
|
class DailyTransportService(EventHandler):
|
||||||
_daily_initialized = False
|
_daily_initialized = False
|
||||||
_lock = threading.Lock()
|
_lock = threading.Lock()
|
||||||
@@ -111,7 +112,8 @@ class DailyTransportService(EventHandler):
|
|||||||
if self.loop:
|
if self.loop:
|
||||||
asyncio.run_coroutine_threadsafe(handler(*args, **kwargs), self.loop)
|
asyncio.run_coroutine_threadsafe(handler(*args, **kwargs), self.loop)
|
||||||
else:
|
else:
|
||||||
raise Exception("No event loop to run coroutine. In order to use async event handlers, you must run the DailyTransportService in an asyncio event loop.")
|
raise Exception(
|
||||||
|
"No event loop to run coroutine. In order to use async event handlers, you must run the DailyTransportService in an asyncio event loop.")
|
||||||
else:
|
else:
|
||||||
handler(*args, **kwargs)
|
handler(*args, **kwargs)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -126,8 +128,11 @@ class DailyTransportService(EventHandler):
|
|||||||
if event_name not in [method[0] for method in methods]:
|
if event_name not in [method[0] for method in methods]:
|
||||||
raise Exception(f"Event handler {event_name} not found")
|
raise Exception(f"Event handler {event_name} not found")
|
||||||
|
|
||||||
if not event_name in self.event_handlers:
|
if event_name not in self.event_handlers:
|
||||||
self.event_handlers[event_name] = [getattr(self, event_name), types.MethodType(handler, self)]
|
self.event_handlers[event_name] = [
|
||||||
|
getattr(
|
||||||
|
self, event_name), types.MethodType(
|
||||||
|
handler, self)]
|
||||||
setattr(self, event_name, partial(self.patch_method, event_name))
|
setattr(self, event_name, partial(self.patch_method, event_name))
|
||||||
else:
|
else:
|
||||||
self.event_handlers[event_name].append(types.MethodType(handler, self))
|
self.event_handlers[event_name].append(types.MethodType(handler, self))
|
||||||
@@ -317,7 +322,7 @@ class DailyTransportService(EventHandler):
|
|||||||
def on_app_message(self, message, sender):
|
def on_app_message(self, message, sender):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def on_transcription_message(self, message:dict):
|
def on_transcription_message(self, message: dict):
|
||||||
if self.loop:
|
if self.loop:
|
||||||
participantId = ""
|
participantId = ""
|
||||||
if "participantId" in message:
|
if "participantId" in message:
|
||||||
@@ -360,7 +365,7 @@ class DailyTransportService(EventHandler):
|
|||||||
frames_or_frame: QueueFrame | list[QueueFrame] = self.threadsafe_send_queue.get()
|
frames_or_frame: QueueFrame | list[QueueFrame] = self.threadsafe_send_queue.get()
|
||||||
if isinstance(frames_or_frame, QueueFrame):
|
if isinstance(frames_or_frame, QueueFrame):
|
||||||
frames: list[QueueFrame] = [frames_or_frame]
|
frames: list[QueueFrame] = [frames_or_frame]
|
||||||
elif isinstance(frames_or_frame, list):
|
elif isinstance(frames_or_frame, list):
|
||||||
frames: list[QueueFrame] = frames_or_frame
|
frames: list[QueueFrame] = frames_or_frame
|
||||||
else:
|
else:
|
||||||
raise Exception("Unknown type in output queue")
|
raise Exception("Unknown type in output queue")
|
||||||
|
|||||||
@@ -7,13 +7,14 @@ import requests
|
|||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from dailyai.services.ai_services import TTSService
|
from dailyai.services.ai_services import TTSService
|
||||||
|
|
||||||
|
|
||||||
class DeepgramTTSService(TTSService):
|
class DeepgramTTSService(TTSService):
|
||||||
def __init__(self, speech_key=None, voice=None):
|
def __init__(self, speech_key=None, voice=None):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
|
||||||
self.voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2"
|
self.voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2"
|
||||||
self.speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY")
|
self.speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY")
|
||||||
|
|
||||||
def get_mic_sample_rate(self):
|
def get_mic_sample_rate(self):
|
||||||
return 24000
|
return 24000
|
||||||
|
|
||||||
@@ -22,8 +23,8 @@ class DeepgramTTSService(TTSService):
|
|||||||
base_url = "https://api.beta.deepgram.com/v1/speak"
|
base_url = "https://api.beta.deepgram.com/v1/speak"
|
||||||
request_url = f"{base_url}?model={self.voice}&encoding=linear16&container=none&sample_rate=16000"
|
request_url = f"{base_url}?model={self.voice}&encoding=linear16&container=none&sample_rate=16000"
|
||||||
headers = {"authorization": f"token {self.speech_key}"}
|
headers = {"authorization": f"token {self.speech_key}"}
|
||||||
body = { "text": sentence }
|
body = {"text": sentence}
|
||||||
async with aiohttp.ClientSession() as session:
|
async with aiohttp.ClientSession() as session:
|
||||||
async with session.post(request_url, headers=headers, json=body) as r:
|
async with session.post(request_url, headers=headers, json=body) as r:
|
||||||
async for data in r.content:
|
async for data in r.content:
|
||||||
yield data
|
yield data
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ from PIL import Image
|
|||||||
|
|
||||||
from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
|
from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
|
||||||
# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
|
# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
|
||||||
|
|
||||||
|
|
||||||
class FalImageGenService(ImageGenService):
|
class FalImageGenService(ImageGenService):
|
||||||
def __init__(self, image_size):
|
def __init__(self, image_size):
|
||||||
super().__init__(image_size)
|
super().__init__(image_size)
|
||||||
@@ -18,9 +20,9 @@ class FalImageGenService(ImageGenService):
|
|||||||
handler = fal.apps.submit(
|
handler = fal.apps.submit(
|
||||||
"110602490-fast-sdxl",
|
"110602490-fast-sdxl",
|
||||||
arguments={
|
arguments={
|
||||||
"prompt": sentence
|
"prompt": sentence
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
print("past fal handler init, about to wait for iter_events...")
|
print("past fal handler init, about to wait for iter_events...")
|
||||||
for event in handler.iter_events():
|
for event in handler.iter_events():
|
||||||
if isinstance(event, fal.apps.InProgress):
|
if isinstance(event, fal.apps.InProgress):
|
||||||
|
|||||||
@@ -49,8 +49,9 @@ class OpenAILLMService(LLMService):
|
|||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
class OpenAIImageGenService(ImageGenService):
|
class OpenAIImageGenService(ImageGenService):
|
||||||
def __init__(self, image_size:str, api_key=None, model=None):
|
def __init__(self, image_size: str, api_key=None, model=None):
|
||||||
super().__init__(image_size=image_size)
|
super().__init__(image_size=image_size)
|
||||||
api_key = api_key or os.getenv("OPEN_AI_KEY")
|
api_key = api_key or os.getenv("OPEN_AI_KEY")
|
||||||
self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3"
|
self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3"
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ from services.ai_service import AIService
|
|||||||
|
|
||||||
# Note that Cloudflare's AI workers are still in beta.
|
# Note that Cloudflare's AI workers are still in beta.
|
||||||
# https://developers.cloudflare.com/workers-ai/
|
# https://developers.cloudflare.com/workers-ai/
|
||||||
|
|
||||||
|
|
||||||
class CloudflareAIService(AIService):
|
class CloudflareAIService(AIService):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -19,11 +21,11 @@ class CloudflareAIService(AIService):
|
|||||||
return response.json()
|
return response.json()
|
||||||
|
|
||||||
# https://developers.cloudflare.com/workers-ai/models/llm/
|
# https://developers.cloudflare.com/workers-ai/models/llm/
|
||||||
def run_llm(self, messages, latest_user_message=None, stream = True):
|
def run_llm(self, messages, latest_user_message=None, stream=True):
|
||||||
input = {
|
input = {
|
||||||
"messages": [
|
"messages": [
|
||||||
{ "role": "system", "content": "You are a friendly assistant" },
|
{"role": "system", "content": "You are a friendly assistant"},
|
||||||
{ "role": "user", "content": sentence }
|
{"role": "user", "content": sentence}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -57,9 +59,9 @@ class CloudflareAIService(AIService):
|
|||||||
# https://developers.cloudflare.com/workers-ai/models/embedding/
|
# https://developers.cloudflare.com/workers-ai/models/embedding/
|
||||||
def run_embeddings(self, texts, size="medium"):
|
def run_embeddings(self, texts, size="medium"):
|
||||||
models = {
|
models = {
|
||||||
"small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions
|
"small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions
|
||||||
"medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions
|
"medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions
|
||||||
"large": "@cf/baai/bge-large-en-v1.5" #1024 output dimensions
|
"large": "@cf/baai/bge-large-en-v1.5" # 1024 output dimensions
|
||||||
}
|
}
|
||||||
|
|
||||||
return self.run(models[size], {"text": texts})
|
return self.run(models[size], {"text": texts})
|
||||||
|
|||||||
@@ -17,7 +17,8 @@ class DeepgramAIService(AIService):
|
|||||||
def run_tts(self, sentence):
|
def run_tts(self, sentence):
|
||||||
self.logger.info(f"Running deepgram tts for {sentence}")
|
self.logger.info(f"Running deepgram tts for {sentence}")
|
||||||
base_url = "https://api.beta.deepgram.com/v1/speak"
|
base_url = "https://api.beta.deepgram.com/v1/speak"
|
||||||
voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1" # move this to an environment variable
|
# move this to an environment variable
|
||||||
|
voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1"
|
||||||
request_url = f"{base_url}?model={voice}&encoding=linear16&container=none"
|
request_url = f"{base_url}?model={voice}&encoding=linear16&container=none"
|
||||||
headers = {"authorization": f"token {self.api_key}"}
|
headers = {"authorization": f"token {self.api_key}"}
|
||||||
|
|
||||||
|
|||||||
@@ -2,9 +2,12 @@ from services.ai_service import AIService
|
|||||||
import openai
|
import openai
|
||||||
import os
|
import os
|
||||||
|
|
||||||
# To use Google Cloud's AI products, you'll need to install Google Cloud CLI and enable the TTS and in your project: https://cloud.google.com/sdk/docs/install
|
# To use Google Cloud's AI products, you'll need to install Google Cloud
|
||||||
|
# CLI and enable the TTS and in your project:
|
||||||
|
# https://cloud.google.com/sdk/docs/install
|
||||||
from google.cloud import texttospeech
|
from google.cloud import texttospeech
|
||||||
|
|
||||||
|
|
||||||
class GoogleAIService(AIService):
|
class GoogleAIService(AIService):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -15,11 +18,14 @@ class GoogleAIService(AIService):
|
|||||||
)
|
)
|
||||||
|
|
||||||
self.audio_config = texttospeech.AudioConfig(
|
self.audio_config = texttospeech.AudioConfig(
|
||||||
audio_encoding = texttospeech.AudioEncoding.LINEAR16,
|
audio_encoding=texttospeech.AudioEncoding.LINEAR16,
|
||||||
sample_rate_hertz = 16000
|
sample_rate_hertz=16000
|
||||||
)
|
)
|
||||||
|
|
||||||
def run_tts(self, sentence):
|
def run_tts(self, sentence):
|
||||||
synthesis_input = texttospeech.SynthesisInput(text = sentence.strip())
|
synthesis_input = texttospeech.SynthesisInput(text=sentence.strip())
|
||||||
result = self.client.synthesize_speech(input=synthesis_input, voice=self.voice, audio_config=self.audio_config)
|
result = self.client.synthesize_speech(
|
||||||
|
input=synthesis_input,
|
||||||
|
voice=self.voice,
|
||||||
|
audio_config=self.audio_config)
|
||||||
return result
|
return result
|
||||||
|
|||||||
@@ -1,7 +1,12 @@
|
|||||||
from services.ai_service import AIService
|
from services.ai_service import AIService
|
||||||
from transformers import pipeline
|
from transformers import pipeline
|
||||||
|
|
||||||
# These functions are just intended for testing, not production use. If you'd like to use HuggingFace, you should use your own models, or do some research into the specific models that will work best for your use case.
|
# These functions are just intended for testing, not production use. If
|
||||||
|
# you'd like to use HuggingFace, you should use your own models, or do
|
||||||
|
# some research into the specific models that will work best for your use
|
||||||
|
# case.
|
||||||
|
|
||||||
|
|
||||||
class HuggingFaceAIService(AIService):
|
class HuggingFaceAIService(AIService):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -10,9 +15,12 @@ class HuggingFaceAIService(AIService):
|
|||||||
classifier = pipeline("sentiment-analysis")
|
classifier = pipeline("sentiment-analysis")
|
||||||
return classifier(sentence)
|
return classifier(sentence)
|
||||||
|
|
||||||
# available models at https://huggingface.co/Helsinki-NLP (**not all models use 2-character language codes**)
|
# available models at https://huggingface.co/Helsinki-NLP (**not all
|
||||||
|
# models use 2-character language codes**)
|
||||||
def run_text_translation(self, sentence, source_language, target_language):
|
def run_text_translation(self, sentence, source_language, target_language):
|
||||||
translator = pipeline(f"translation", model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
|
translator = pipeline(
|
||||||
|
f"translation",
|
||||||
|
model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
|
||||||
|
|
||||||
return translator(sentence)[0]["translation_text"]
|
return translator(sentence)[0]["translation_text"]
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import time
|
|||||||
from PIL import Image
|
from PIL import Image
|
||||||
from services.ai_service import AIService
|
from services.ai_service import AIService
|
||||||
|
|
||||||
|
|
||||||
class MockAIService(AIService):
|
class MockAIService(AIService):
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -20,8 +21,7 @@ class MockAIService(AIService):
|
|||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
return (image_url, image)
|
return (image_url, image)
|
||||||
|
|
||||||
def run_llm(self, messages, latest_user_message=None, stream = True):
|
def run_llm(self, messages, latest_user_message=None, stream=True):
|
||||||
for i in range(5):
|
for i in range(5):
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
yield({"choices": [{"delta": {"content": f"hello {i}!"}}]})
|
yield ({"choices": [{"delta": {"content": f"hello {i}!"}}]})
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from pyht.protos.api_pb2 import Format
|
|||||||
|
|
||||||
from services.ai_service import AIService
|
from services.ai_service import AIService
|
||||||
|
|
||||||
|
|
||||||
class PlayHTAIService(AIService):
|
class PlayHTAIService(AIService):
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
@@ -23,8 +24,7 @@ class PlayHTAIService(AIService):
|
|||||||
voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json",
|
voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json",
|
||||||
sample_rate=16000,
|
sample_rate=16000,
|
||||||
quality="higher",
|
quality="higher",
|
||||||
format=Format.FORMAT_WAV
|
format=Format.FORMAT_WAV)
|
||||||
)
|
|
||||||
|
|
||||||
def close(self):
|
def close(self):
|
||||||
super().close()
|
super().close()
|
||||||
@@ -43,14 +43,15 @@ class PlayHTAIService(AIService):
|
|||||||
fh = io.BytesIO(b)
|
fh = io.BytesIO(b)
|
||||||
fh.seek(36)
|
fh.seek(36)
|
||||||
(data, size) = struct.unpack('<4sI', fh.read(8))
|
(data, size) = struct.unpack('<4sI', fh.read(8))
|
||||||
self.logger.info(f"first attempt: data: {data}, size: {hex(size)}, position: {fh.tell()}")
|
self.logger.info(
|
||||||
|
f"first attempt: data: {data}, size: {hex(size)}, position: {fh.tell()}")
|
||||||
while data != b'data':
|
while data != b'data':
|
||||||
fh.read(size)
|
fh.read(size)
|
||||||
(data, size) = struct.unpack('<4sI', fh.read(8))
|
(data, size) = struct.unpack('<4sI', fh.read(8))
|
||||||
self.logger.info(f"subsequent data: {data}, size: {hex(size)}, position: {fh.tell()}, data != data: {data != b'data'}")
|
self.logger.info(
|
||||||
|
f"subsequent data: {data}, size: {hex(size)}, position: {fh.tell()}, data != data: {data != b'data'}")
|
||||||
self.logger.info("position: ", fh.tell())
|
self.logger.info("position: ", fh.tell())
|
||||||
in_header = False
|
in_header = False
|
||||||
else:
|
else:
|
||||||
if len(chunk):
|
if len(chunk):
|
||||||
yield chunk
|
yield chunk
|
||||||
|
|
||||||
|
|||||||
@@ -6,10 +6,12 @@ from typing import AsyncGenerator, Generator
|
|||||||
from dailyai.services.ai_services import AIService
|
from dailyai.services.ai_services import AIService
|
||||||
from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TextQueueFrame
|
from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TextQueueFrame
|
||||||
|
|
||||||
|
|
||||||
class SimpleAIService(AIService):
|
class SimpleAIService(AIService):
|
||||||
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
|
||||||
yield frame
|
yield frame
|
||||||
|
|
||||||
|
|
||||||
class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
|
class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
|
||||||
async def test_async_input(self):
|
async def test_async_input(self):
|
||||||
service = SimpleAIService()
|
service = SimpleAIService()
|
||||||
@@ -18,6 +20,7 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
|
|||||||
TextQueueFrame("hello"),
|
TextQueueFrame("hello"),
|
||||||
EndStreamQueueFrame()
|
EndStreamQueueFrame()
|
||||||
]
|
]
|
||||||
|
|
||||||
async def iterate_frames() -> AsyncGenerator[QueueFrame, None]:
|
async def iterate_frames() -> AsyncGenerator[QueueFrame, None]:
|
||||||
for frame in input_frames:
|
for frame in input_frames:
|
||||||
yield frame
|
yield frame
|
||||||
|
|||||||
@@ -15,11 +15,12 @@ from dailyai.services.ai_services import AIServiceConfig
|
|||||||
from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
|
from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
|
||||||
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
||||||
|
|
||||||
|
|
||||||
def add_bot_to_room(room_url, token, expiration) -> None:
|
def add_bot_to_room(room_url, token, expiration) -> None:
|
||||||
|
|
||||||
# A simple prompt for a simple sample.
|
# A simple prompt for a simple sample.
|
||||||
message_handler = MessageHandler(
|
message_handler = MessageHandler(
|
||||||
"""
|
"""
|
||||||
You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's
|
You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's
|
||||||
speech, and your responses will be converted to audio via a TTS service.
|
speech, and your responses will be converted to audio via a TTS service.
|
||||||
Answer user's questions and be friendly, and if you can, give some ideas about how someone
|
Answer user's questions and be friendly, and if you can, give some ideas about how someone
|
||||||
@@ -62,6 +63,7 @@ def add_bot_to_room(room_url, token, expiration) -> None:
|
|||||||
services.tts.close()
|
services.tts.close()
|
||||||
services.llm.close()
|
services.llm.close()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
||||||
parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room")
|
parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room")
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from dailyai.message_handler.message_handler import MessageHandler
|
|||||||
from dailyai.services.ai_services import AIServiceConfig
|
from dailyai.services.ai_services import AIServiceConfig
|
||||||
from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
|
from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
|
||||||
|
|
||||||
|
|
||||||
class StaticSpriteResponse(OrchestratorResponse):
|
class StaticSpriteResponse(OrchestratorResponse):
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
@@ -29,8 +30,8 @@ class StaticSpriteResponse(OrchestratorResponse):
|
|||||||
output_queue
|
output_queue
|
||||||
) -> None:
|
) -> None:
|
||||||
super().__init__(services, message_handler, output_queue)
|
super().__init__(services, message_handler, output_queue)
|
||||||
self.image_bytes:bytes | None = None
|
self.image_bytes: bytes | None = None
|
||||||
self.filenames = None # override this in subclasses
|
self.filenames = None # override this in subclasses
|
||||||
|
|
||||||
def start_preparation(self) -> None:
|
def start_preparation(self) -> None:
|
||||||
full_path = os.path.join(os.path.dirname(__file__), "sprites/", self.filename)
|
full_path = os.path.join(os.path.dirname(__file__), "sprites/", self.filename)
|
||||||
@@ -82,7 +83,7 @@ def add_bot_to_room(room_url, token, expiration) -> None:
|
|||||||
|
|
||||||
# A simple prompt for a simple sample.
|
# A simple prompt for a simple sample.
|
||||||
message_handler = MessageHandler(
|
message_handler = MessageHandler(
|
||||||
"""
|
"""
|
||||||
You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's
|
You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's
|
||||||
speech, and your responses will be converted to audio via a TTS service.
|
speech, and your responses will be converted to audio via a TTS service.
|
||||||
Answer user's questions and be friendly, and if you can, give some ideas about how someone
|
Answer user's questions and be friendly, and if you can, give some ideas about how someone
|
||||||
@@ -143,6 +144,7 @@ def add_bot_to_room(room_url, token, expiration) -> None:
|
|||||||
services.image.close()
|
services.image.close()
|
||||||
services.llm.close()
|
services.llm.close()
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
||||||
parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room")
|
parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room")
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import asyncio
|
|||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
# create a transport service object using environment variables for
|
# create a transport service object using environment variables for
|
||||||
# the transport service's API key, room url, and any other configuration.
|
# the transport service's API key, room url, and any other configuration.
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from dailyai.services.daily_transport_service import DailyTransportService
|
|||||||
from dailyai.services.azure_ai_services import AzureTTSService
|
from dailyai.services.azure_ai_services import AzureTTSService
|
||||||
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
from dailyai.services.deepgram_ai_services import DeepgramTTSService
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
# create a transport service object using environment variables for
|
# create a transport service object using environment variables for
|
||||||
# the transport service's API key, room url, and any other configuration.
|
# the transport service's API key, room url, and any other configuration.
|
||||||
@@ -33,17 +34,20 @@ async def main(room_url):
|
|||||||
|
|
||||||
# Register an event handler so we can play the audio when the participant joins.
|
# Register an event handler so we can play the audio when the participant joins.
|
||||||
print("settting up handler")
|
print("settting up handler")
|
||||||
|
|
||||||
@transport.event_handler("on_participant_joined")
|
@transport.event_handler("on_participant_joined")
|
||||||
async def on_participant_joined(transport, participant):
|
async def on_participant_joined(transport, participant):
|
||||||
print(f"participant joined: {participant['info']['userName']}")
|
print(f"participant joined: {participant['info']['userName']}")
|
||||||
if participant["info"]["isLocal"]:
|
if participant["info"]["isLocal"]:
|
||||||
return
|
return
|
||||||
audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(f"Hello there, {participant['info']['userName']}!")
|
audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(
|
||||||
|
f"Hello there, {participant['info']['userName']}!")
|
||||||
|
|
||||||
async for audio in audio_generator:
|
async for audio in audio_generator:
|
||||||
transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio))
|
transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio))
|
||||||
|
|
||||||
print("setting up call state handler")
|
print("setting up call state handler")
|
||||||
|
|
||||||
@transport.event_handler("on_call_state_updated")
|
@transport.event_handler("on_call_state_updated")
|
||||||
async def on_call_joined(transport, state):
|
async def on_call_joined(transport, state):
|
||||||
print(f"call state callback: {state}")
|
print(f"call state callback: {state}")
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from dailyai.services.daily_transport_service import DailyTransportService
|
|||||||
from dailyai.services.azure_ai_services import AzureLLMService
|
from dailyai.services.azure_ai_services import AzureLLMService
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
meeting_duration_minutes = 1
|
meeting_duration_minutes = 1
|
||||||
transport = DailyTransportService(
|
transport = DailyTransportService(
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from dailyai.services.open_ai_services import OpenAIImageGenService
|
|||||||
local_joined = False
|
local_joined = False
|
||||||
participant_joined = False
|
participant_joined = False
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
meeting_duration_minutes = 1
|
meeting_duration_minutes = 1
|
||||||
transport = DailyTransportService(
|
transport = DailyTransportService(
|
||||||
@@ -23,8 +24,9 @@ async def main(room_url):
|
|||||||
|
|
||||||
imagegen = OpenAIImageGenService(image_size="1024x1024")
|
imagegen = OpenAIImageGenService(image_size="1024x1024")
|
||||||
image_task = asyncio.create_task(
|
image_task = asyncio.create_task(
|
||||||
imagegen.run_to_queue(transport.send_queue, [TextQueueFrame("a cat in the style of picasso")])
|
imagegen.run_to_queue(
|
||||||
)
|
transport.send_queue, [
|
||||||
|
TextQueueFrame("a cat in the style of picasso")]))
|
||||||
|
|
||||||
@transport.event_handler("on_participant_joined")
|
@transport.event_handler("on_participant_joined")
|
||||||
async def on_participant_joined(transport, participant):
|
async def on_participant_joined(transport, participant):
|
||||||
|
|||||||
@@ -7,7 +7,8 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
|||||||
from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame
|
from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
|
||||||
async def main(room_url:str):
|
|
||||||
|
async def main(room_url: str):
|
||||||
global transport
|
global transport
|
||||||
global llm
|
global llm
|
||||||
global tts
|
global tts
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
|||||||
from dailyai.services.daily_transport_service import DailyTransportService
|
from dailyai.services.daily_transport_service import DailyTransportService
|
||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url):
|
async def main(room_url):
|
||||||
meeting_duration_minutes = 5
|
meeting_duration_minutes = 5
|
||||||
transport = DailyTransportService(
|
transport = DailyTransportService(
|
||||||
@@ -98,7 +99,7 @@ async def main(room_url):
|
|||||||
|
|
||||||
await transport.run()
|
await transport.run()
|
||||||
|
|
||||||
if __name__=="__main__":
|
if __name__ == "__main__":
|
||||||
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
|
"-u", "--url", type=str, required=True, help="URL of the Daily room to join"
|
||||||
|
|||||||
@@ -8,7 +8,8 @@ from dailyai.services.daily_transport_service import DailyTransportService
|
|||||||
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
|
||||||
from dailyai.queue_aggregators import LLMContextAggregator
|
from dailyai.queue_aggregators import LLMContextAggregator
|
||||||
|
|
||||||
async def main(room_url:str, token):
|
|
||||||
|
async def main(room_url: str, token):
|
||||||
global transport
|
global transport
|
||||||
global llm
|
global llm
|
||||||
global tts
|
global tts
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from dailyai.services.azure_ai_services import AzureLLMService
|
|||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
|
||||||
|
|
||||||
async def main(room_url:str, token):
|
async def main(room_url: str, token):
|
||||||
global transport
|
global transport
|
||||||
global llm
|
global llm
|
||||||
global tts
|
global tts
|
||||||
|
|||||||
@@ -11,7 +11,8 @@ from dailyai.queue_frame import QueueFrame, FrameType
|
|||||||
from dailyai.services.fal_ai_services import FalImageGenService
|
from dailyai.services.fal_ai_services import FalImageGenService
|
||||||
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
|
||||||
|
|
||||||
async def main(room_url:str, token):
|
|
||||||
|
async def main(room_url: str, token):
|
||||||
global transport
|
global transport
|
||||||
global llm
|
global llm
|
||||||
global tts
|
global tts
|
||||||
@@ -32,7 +33,6 @@ async def main(room_url:str, token):
|
|||||||
tts = AzureTTSService()
|
tts = AzureTTSService()
|
||||||
img = FalImageGenService()
|
img = FalImageGenService()
|
||||||
|
|
||||||
|
|
||||||
async def handle_transcriptions():
|
async def handle_transcriptions():
|
||||||
print("handle_transcriptions got called")
|
print("handle_transcriptions got called")
|
||||||
|
|
||||||
@@ -41,7 +41,7 @@ async def main(room_url:str, token):
|
|||||||
print(f"transcription message: {message}")
|
print(f"transcription message: {message}")
|
||||||
if message["session_id"] == transport.my_participant_id:
|
if message["session_id"] == transport.my_participant_id:
|
||||||
continue
|
continue
|
||||||
finder = message["text"].find("start over")
|
finder = message["text"].find("start over")
|
||||||
print(f"finder: {finder}")
|
print(f"finder: {finder}")
|
||||||
if finder >= 0:
|
if finder >= 0:
|
||||||
async for audio in tts.run_tts(f"Resetting."):
|
async for audio in tts.run_tts(f"Resetting."):
|
||||||
@@ -69,7 +69,8 @@ async def main(room_url:str, token):
|
|||||||
if participant["info"]["isLocal"]:
|
if participant["info"]["isLocal"]:
|
||||||
return
|
return
|
||||||
async for audio in tts.run_tts("Describe an image, and I'll create it."):
|
async for audio in tts.run_tts("Describe an image, and I'll create it."):
|
||||||
audio_generator = tts.run_tts(f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.")
|
audio_generator = tts.run_tts(
|
||||||
|
f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.")
|
||||||
async for audio in audio_generator:
|
async for audio in audio_generator:
|
||||||
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
|
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user