Merge pull request #11 from daily-co/autopep

Autopep linter fixes
This commit is contained in:
Moishe Lettvin
2024-01-25 12:17:26 -05:00
committed by GitHub
27 changed files with 147 additions and 68 deletions

View File

@@ -5,6 +5,7 @@ from typing import AsyncGenerator, Awaitable, Callable
from dailyai.queue_aggregators import LLMContextAggregator from dailyai.queue_aggregators import LLMContextAggregator
from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TranscriptionQueueFrame from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TranscriptionQueueFrame
class InterruptibleConversationWrapper: class InterruptibleConversationWrapper:
def __init__( def __init__(
@@ -14,7 +15,7 @@ class InterruptibleConversationWrapper:
[str, LLMContextAggregator, LLMContextAggregator], Awaitable[None] [str, LLMContextAggregator, LLMContextAggregator], Awaitable[None]
], ],
interrupt: Callable[[], None], interrupt: Callable[[], None],
my_participant_id: str|None, my_participant_id: str | None,
llm_messages: list[dict[str, str]], llm_messages: list[dict[str, str]],
llm_context_aggregator_in=LLMContextAggregator, llm_context_aggregator_in=LLMContextAggregator,
llm_context_aggregator_out=LLMContextAggregator, llm_context_aggregator_out=LLMContextAggregator,

View File

@@ -5,6 +5,7 @@ from dailyai.services.ai_services import AIService
from typing import AsyncGenerator, List from typing import AsyncGenerator, List
class QueueTee: class QueueTee:
async def run_to_queue_and_generate( async def run_to_queue_and_generate(
self, self,
@@ -24,15 +25,21 @@ class QueueTee:
for queue in output_queues: for queue in output_queues:
await queue.put(frame) await queue.put(frame)
class LLMContextAggregator(AIService): class LLMContextAggregator(AIService):
def __init__(self, messages: list[dict], role:str, bot_participant_id=None, complete_sentences=True): def __init__(
self,
messages: list[dict],
role: str,
bot_participant_id=None,
complete_sentences=True):
self.messages = messages self.messages = messages
self.bot_participant_id = bot_participant_id self.bot_participant_id = bot_participant_id
self.role = role self.role = role
self.sentence = "" self.sentence = ""
self.complete_sentences = complete_sentences self.complete_sentences = complete_sentences
async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
# TODO: split up transcription by participant # TODO: split up transcription by participant
if isinstance(frame, TextQueueFrame): if isinstance(frame, TextQueueFrame):
if self.complete_sentences: if self.complete_sentences:

View File

@@ -2,39 +2,49 @@ from enum import Enum
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any from typing import Any
class QueueFrame: class QueueFrame:
pass pass
class ControlQueueFrame(QueueFrame): class ControlQueueFrame(QueueFrame):
pass pass
class StartStreamQueueFrame(ControlQueueFrame): class StartStreamQueueFrame(ControlQueueFrame):
pass pass
class EndStreamQueueFrame(ControlQueueFrame): class EndStreamQueueFrame(ControlQueueFrame):
pass pass
@dataclass() @dataclass()
class AudioQueueFrame(QueueFrame): class AudioQueueFrame(QueueFrame):
data: bytes data: bytes
@dataclass() @dataclass()
class ImageQueueFrame(QueueFrame): class ImageQueueFrame(QueueFrame):
url: str | None url: str | None
image: bytes image: bytes
@dataclass() @dataclass()
class TextQueueFrame(QueueFrame): class TextQueueFrame(QueueFrame):
text: str text: str
@dataclass() @dataclass()
class TranscriptionQueueFrame(TextQueueFrame): class TranscriptionQueueFrame(TextQueueFrame):
participantId: str participantId: str
timestamp: str timestamp: str
@dataclass() @dataclass()
class LLMMessagesQueueFrame(QueueFrame): class LLMMessagesQueueFrame(QueueFrame):
messages: list[dict[str,str]] # TODO: define this more concretely! messages: list[dict[str, str]] # TODO: define this more concretely!
class AppMessageQueueFrame(QueueFrame): class AppMessageQueueFrame(QueueFrame):
message: Any message: Any

View File

@@ -65,7 +65,7 @@ class AIService:
raise e raise e
@abstractmethod @abstractmethod
async def process_frame(self, frame:QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
if isinstance(frame, ControlQueueFrame): if isinstance(frame, ControlQueueFrame):
yield frame yield frame
@@ -75,6 +75,7 @@ class AIService:
if False: if False:
yield QueueFrame() yield QueueFrame()
class LLMService(AIService): class LLMService(AIService):
@abstractmethod @abstractmethod
async def run_llm_async(self, messages) -> AsyncGenerator[str, None]: async def run_llm_async(self, messages) -> AsyncGenerator[str, None]:
@@ -146,7 +147,7 @@ class ImageGenService(AIService):
# Renders the image. Returns an Image object. # Renders the image. Returns an Image object.
@abstractmethod @abstractmethod
async def run_image_gen(self, sentence:str) -> tuple[str, bytes]: async def run_image_gen(self, sentence: str) -> tuple[str, bytes]:
pass pass
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
@@ -157,15 +158,16 @@ class ImageGenService(AIService):
(url, image_data) = await self.run_image_gen(frame.text) (url, image_data) = await self.run_image_gen(frame.text)
yield ImageQueueFrame(url, image_data) yield ImageQueueFrame(url, image_data)
class STTService(AIService): class STTService(AIService):
"""STTService is a base class for speech-to-text services.""" """STTService is a base class for speech-to-text services."""
_frame_rate: int _frame_rate: int
def __init__(self, frame_rate: int = 16000, **kwargs): def __init__(self, frame_rate: int = 16000, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
self._frame_rate = frame_rate self._frame_rate = frame_rate
@abstractmethod @abstractmethod
async def run_stt(self, audio: BinaryIO) -> str: async def run_stt(self, audio: BinaryIO) -> str:
"""Returns transcript as a string""" """Returns transcript as a string"""
@@ -188,6 +190,7 @@ class STTService(AIService):
text = await self.run_stt(content) text = await self.run_stt(content)
yield TextQueueFrame(text) yield TextQueueFrame(text)
@dataclass @dataclass
class AIServiceConfig: class AIServiceConfig:
tts: TTSService tts: TTSService

View File

@@ -15,6 +15,7 @@ from PIL import Image
# See .env.example for Azure configuration needed # See .env.example for Azure configuration needed
from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason from azure.cognitiveservices.speech import SpeechSynthesizer, SpeechConfig, ResultReason, CancellationReason
class AzureTTSService(TTSService): class AzureTTSService(TTSService):
def __init__(self, speech_key=None, speech_region=None): def __init__(self, speech_key=None, speech_region=None):
super().__init__() super().__init__()
@@ -23,18 +24,19 @@ class AzureTTSService(TTSService):
speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION") speech_region = speech_region or os.getenv("AZURE_SPEECH_SERVICE_REGION")
self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region) self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region)
self.speech_synthesizer = SpeechSynthesizer(speech_config=self.speech_config, audio_config=None) self.speech_synthesizer = SpeechSynthesizer(
speech_config=self.speech_config, audio_config=None)
async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]: async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]:
self.logger.info("Running azure tts") self.logger.info("Running azure tts")
ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \ ssml = "<speak version='1.0' xml:lang='en-US' xmlns='http://www.w3.org/2001/10/synthesis' " \
"xmlns:mstts='http://www.w3.org/2001/mstts'>" \ "xmlns:mstts='http://www.w3.org/2001/mstts'>" \
"<voice name='en-US-SaraNeural'>" \ "<voice name='en-US-SaraNeural'>" \
"<mstts:silence type='Sentenceboundary' value='20ms' />" \ "<mstts:silence type='Sentenceboundary' value='20ms' />" \
"<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>" \ "<mstts:express-as style='lyrical' styledegree='2' role='SeniorFemale'>" \
"<prosody rate='1.05'>" \ "<prosody rate='1.05'>" \
f"{sentence}" \ f"{sentence}" \
"</prosody></mstts:express-as></voice></speak> " "</prosody></mstts:express-as></voice></speak> "
result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml)) result = await asyncio.to_thread(self.speech_synthesizer.speak_ssml, (ssml))
self.logger.info("Got azure tts result") self.logger.info("Got azure tts result")
if result.reason == ResultReason.SynthesizingAudioCompleted: if result.reason == ResultReason.SynthesizingAudioCompleted:
@@ -47,6 +49,7 @@ class AzureTTSService(TTSService):
if cancellation_details.reason == CancellationReason.Error: if cancellation_details.reason == CancellationReason.Error:
self.logger.info("Error details: {}".format(cancellation_details.error_details)) self.logger.info("Error details: {}".format(cancellation_details.error_details))
class AzureLLMService(LLMService): class AzureLLMService(LLMService):
def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None): def __init__(self, api_key=None, azure_endpoint=None, api_version=None, model=None):
super().__init__() super().__init__()
@@ -54,11 +57,13 @@ class AzureLLMService(LLMService):
azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT") azure_endpoint = azure_endpoint or os.getenv("AZURE_CHATGPT_ENDPOINT")
if not azure_endpoint: if not azure_endpoint:
raise Exception("No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor") raise Exception(
"No azure endpoint specified for Azure LLM, please set AZURE_CHATGPT_ENDPOINT in the environment or pass it to the AzureLLMService constructor")
model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID") model: str | None = model or os.getenv("AZURE_CHATGPT_DEPLOYMENT_ID")
if not model: if not model:
raise Exception("No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor") raise Exception(
"No model specified for Azure LLM, please set AZURE_CHATGPT_DEPLOYMENT_ID in the environment or pass it to the AzureLLMService constructor")
self.model: str = model self.model: str = model
api_version = api_version or "2023-12-01-preview" api_version = api_version or "2023-12-01-preview"
@@ -90,9 +95,16 @@ class AzureLLMService(LLMService):
else: else:
return None return None
class AzureImageGenServiceREST(ImageGenService): class AzureImageGenServiceREST(ImageGenService):
def __init__(self, image_size:str, api_key=None, azure_endpoint=None, api_version=None, model=None): def __init__(
self,
image_size: str,
api_key=None,
azure_endpoint=None,
api_version=None,
model=None):
super().__init__(image_size=image_size) super().__init__(image_size=image_size)
self.api_key = api_key or os.getenv("AZURE_DALLE_KEY") self.api_key = api_key or os.getenv("AZURE_DALLE_KEY")
self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT") self.azure_endpoint = azure_endpoint or os.getenv("AZURE_DALLE_ENDPOINT")
@@ -103,7 +115,7 @@ class AzureImageGenServiceREST(ImageGenService):
# TODO hoist the session to app-level # TODO hoist the session to app-level
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}" url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}"
headers= { "api-key": self.api_key, "Content-Type": "application/json" } headers = {"api-key": self.api_key, "Content-Type": "application/json"}
body = { body = {
# Enter your prompt text here # Enter your prompt text here
"prompt": sentence, "prompt": sentence,

View File

@@ -30,6 +30,7 @@ from daily import (
VirtualSpeakerDevice, VirtualSpeakerDevice,
) )
class DailyTransportService(EventHandler): class DailyTransportService(EventHandler):
_daily_initialized = False _daily_initialized = False
_lock = threading.Lock() _lock = threading.Lock()
@@ -111,7 +112,8 @@ class DailyTransportService(EventHandler):
if self.loop: if self.loop:
asyncio.run_coroutine_threadsafe(handler(*args, **kwargs), self.loop) asyncio.run_coroutine_threadsafe(handler(*args, **kwargs), self.loop)
else: else:
raise Exception("No event loop to run coroutine. In order to use async event handlers, you must run the DailyTransportService in an asyncio event loop.") raise Exception(
"No event loop to run coroutine. In order to use async event handlers, you must run the DailyTransportService in an asyncio event loop.")
else: else:
handler(*args, **kwargs) handler(*args, **kwargs)
except Exception as e: except Exception as e:
@@ -126,8 +128,11 @@ class DailyTransportService(EventHandler):
if event_name not in [method[0] for method in methods]: if event_name not in [method[0] for method in methods]:
raise Exception(f"Event handler {event_name} not found") raise Exception(f"Event handler {event_name} not found")
if not event_name in self.event_handlers: if event_name not in self.event_handlers:
self.event_handlers[event_name] = [getattr(self, event_name), types.MethodType(handler, self)] self.event_handlers[event_name] = [
getattr(
self, event_name), types.MethodType(
handler, self)]
setattr(self, event_name, partial(self.patch_method, event_name)) setattr(self, event_name, partial(self.patch_method, event_name))
else: else:
self.event_handlers[event_name].append(types.MethodType(handler, self)) self.event_handlers[event_name].append(types.MethodType(handler, self))
@@ -317,7 +322,7 @@ class DailyTransportService(EventHandler):
def on_app_message(self, message, sender): def on_app_message(self, message, sender):
pass pass
def on_transcription_message(self, message:dict): def on_transcription_message(self, message: dict):
if self.loop: if self.loop:
participantId = "" participantId = ""
if "participantId" in message: if "participantId" in message:
@@ -360,7 +365,7 @@ class DailyTransportService(EventHandler):
frames_or_frame: QueueFrame | list[QueueFrame] = self.threadsafe_send_queue.get() frames_or_frame: QueueFrame | list[QueueFrame] = self.threadsafe_send_queue.get()
if isinstance(frames_or_frame, QueueFrame): if isinstance(frames_or_frame, QueueFrame):
frames: list[QueueFrame] = [frames_or_frame] frames: list[QueueFrame] = [frames_or_frame]
elif isinstance(frames_or_frame, list): elif isinstance(frames_or_frame, list):
frames: list[QueueFrame] = frames_or_frame frames: list[QueueFrame] = frames_or_frame
else: else:
raise Exception("Unknown type in output queue") raise Exception("Unknown type in output queue")

View File

@@ -7,13 +7,14 @@ import requests
from collections.abc import AsyncGenerator from collections.abc import AsyncGenerator
from dailyai.services.ai_services import TTSService from dailyai.services.ai_services import TTSService
class DeepgramTTSService(TTSService): class DeepgramTTSService(TTSService):
def __init__(self, speech_key=None, voice=None): def __init__(self, speech_key=None, voice=None):
super().__init__() super().__init__()
self.voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2" self.voice = voice or os.getenv("DEEPGRAM_VOICE") or "alpha-asteria-en-v2"
self.speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY") self.speech_key = speech_key or os.getenv("DEEPGRAM_API_KEY")
def get_mic_sample_rate(self): def get_mic_sample_rate(self):
return 24000 return 24000
@@ -22,8 +23,8 @@ class DeepgramTTSService(TTSService):
base_url = "https://api.beta.deepgram.com/v1/speak" base_url = "https://api.beta.deepgram.com/v1/speak"
request_url = f"{base_url}?model={self.voice}&encoding=linear16&container=none&sample_rate=16000" request_url = f"{base_url}?model={self.voice}&encoding=linear16&container=none&sample_rate=16000"
headers = {"authorization": f"token {self.speech_key}"} headers = {"authorization": f"token {self.speech_key}"}
body = { "text": sentence } body = {"text": sentence}
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
async with session.post(request_url, headers=headers, json=body) as r: async with session.post(request_url, headers=headers, json=body) as r:
async for data in r.content: async for data in r.content:
yield data yield data

View File

@@ -8,6 +8,8 @@ from PIL import Image
from dailyai.services.ai_services import LLMService, TTSService, ImageGenService from dailyai.services.ai_services import LLMService, TTSService, ImageGenService
# Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env # Fal expects FAL_KEY_ID and FAL_KEY_SECRET to be set in the env
class FalImageGenService(ImageGenService): class FalImageGenService(ImageGenService):
def __init__(self, image_size): def __init__(self, image_size):
super().__init__(image_size) super().__init__(image_size)
@@ -18,9 +20,9 @@ class FalImageGenService(ImageGenService):
handler = fal.apps.submit( handler = fal.apps.submit(
"110602490-fast-sdxl", "110602490-fast-sdxl",
arguments={ arguments={
"prompt": sentence "prompt": sentence
}, },
) )
print("past fal handler init, about to wait for iter_events...") print("past fal handler init, about to wait for iter_events...")
for event in handler.iter_events(): for event in handler.iter_events():
if isinstance(event, fal.apps.InProgress): if isinstance(event, fal.apps.InProgress):

View File

@@ -49,8 +49,9 @@ class OpenAILLMService(LLMService):
else: else:
return None return None
class OpenAIImageGenService(ImageGenService): class OpenAIImageGenService(ImageGenService):
def __init__(self, image_size:str, api_key=None, model=None): def __init__(self, image_size: str, api_key=None, model=None):
super().__init__(image_size=image_size) super().__init__(image_size=image_size)
api_key = api_key or os.getenv("OPEN_AI_KEY") api_key = api_key or os.getenv("OPEN_AI_KEY")
self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3" self.model = model or os.getenv("OPEN_AI_IMAGE_MODEL") or "dall-e-3"

View File

@@ -4,6 +4,8 @@ from services.ai_service import AIService
# Note that Cloudflare's AI workers are still in beta. # Note that Cloudflare's AI workers are still in beta.
# https://developers.cloudflare.com/workers-ai/ # https://developers.cloudflare.com/workers-ai/
class CloudflareAIService(AIService): class CloudflareAIService(AIService):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -19,11 +21,11 @@ class CloudflareAIService(AIService):
return response.json() return response.json()
# https://developers.cloudflare.com/workers-ai/models/llm/ # https://developers.cloudflare.com/workers-ai/models/llm/
def run_llm(self, messages, latest_user_message=None, stream = True): def run_llm(self, messages, latest_user_message=None, stream=True):
input = { input = {
"messages": [ "messages": [
{ "role": "system", "content": "You are a friendly assistant" }, {"role": "system", "content": "You are a friendly assistant"},
{ "role": "user", "content": sentence } {"role": "user", "content": sentence}
] ]
} }
@@ -57,9 +59,9 @@ class CloudflareAIService(AIService):
# https://developers.cloudflare.com/workers-ai/models/embedding/ # https://developers.cloudflare.com/workers-ai/models/embedding/
def run_embeddings(self, texts, size="medium"): def run_embeddings(self, texts, size="medium"):
models = { models = {
"small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions "small": "@cf/baai/bge-small-en-v1.5", # 384 output dimensions
"medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions "medium": "@cf/baai/bge-base-en-v1.5", # 768 output dimensions
"large": "@cf/baai/bge-large-en-v1.5" #1024 output dimensions "large": "@cf/baai/bge-large-en-v1.5" # 1024 output dimensions
} }
return self.run(models[size], {"text": texts}) return self.run(models[size], {"text": texts})

View File

@@ -17,7 +17,8 @@ class DeepgramAIService(AIService):
def run_tts(self, sentence): def run_tts(self, sentence):
self.logger.info(f"Running deepgram tts for {sentence}") self.logger.info(f"Running deepgram tts for {sentence}")
base_url = "https://api.beta.deepgram.com/v1/speak" base_url = "https://api.beta.deepgram.com/v1/speak"
voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1" # move this to an environment variable # move this to an environment variable
voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1"
request_url = f"{base_url}?model={voice}&encoding=linear16&container=none" request_url = f"{base_url}?model={voice}&encoding=linear16&container=none"
headers = {"authorization": f"token {self.api_key}"} headers = {"authorization": f"token {self.api_key}"}

View File

@@ -2,9 +2,12 @@ from services.ai_service import AIService
import openai import openai
import os import os
# To use Google Cloud's AI products, you'll need to install Google Cloud CLI and enable the TTS and in your project: https://cloud.google.com/sdk/docs/install # To use Google Cloud's AI products, you'll need to install Google Cloud
# CLI and enable the TTS and in your project:
# https://cloud.google.com/sdk/docs/install
from google.cloud import texttospeech from google.cloud import texttospeech
class GoogleAIService(AIService): class GoogleAIService(AIService):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -15,11 +18,14 @@ class GoogleAIService(AIService):
) )
self.audio_config = texttospeech.AudioConfig( self.audio_config = texttospeech.AudioConfig(
audio_encoding = texttospeech.AudioEncoding.LINEAR16, audio_encoding=texttospeech.AudioEncoding.LINEAR16,
sample_rate_hertz = 16000 sample_rate_hertz=16000
) )
def run_tts(self, sentence): def run_tts(self, sentence):
synthesis_input = texttospeech.SynthesisInput(text = sentence.strip()) synthesis_input = texttospeech.SynthesisInput(text=sentence.strip())
result = self.client.synthesize_speech(input=synthesis_input, voice=self.voice, audio_config=self.audio_config) result = self.client.synthesize_speech(
input=synthesis_input,
voice=self.voice,
audio_config=self.audio_config)
return result return result

View File

@@ -1,7 +1,12 @@
from services.ai_service import AIService from services.ai_service import AIService
from transformers import pipeline from transformers import pipeline
# These functions are just intended for testing, not production use. If you'd like to use HuggingFace, you should use your own models, or do some research into the specific models that will work best for your use case. # These functions are just intended for testing, not production use. If
# you'd like to use HuggingFace, you should use your own models, or do
# some research into the specific models that will work best for your use
# case.
class HuggingFaceAIService(AIService): class HuggingFaceAIService(AIService):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -10,9 +15,12 @@ class HuggingFaceAIService(AIService):
classifier = pipeline("sentiment-analysis") classifier = pipeline("sentiment-analysis")
return classifier(sentence) return classifier(sentence)
# available models at https://huggingface.co/Helsinki-NLP (**not all models use 2-character language codes**) # available models at https://huggingface.co/Helsinki-NLP (**not all
# models use 2-character language codes**)
def run_text_translation(self, sentence, source_language, target_language): def run_text_translation(self, sentence, source_language, target_language):
translator = pipeline(f"translation", model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}") translator = pipeline(
f"translation",
model=f"Helsinki-NLP/opus-mt-{source_language}-{target_language}")
return translator(sentence)[0]["translation_text"] return translator(sentence)[0]["translation_text"]

View File

@@ -4,6 +4,7 @@ import time
from PIL import Image from PIL import Image
from services.ai_service import AIService from services.ai_service import AIService
class MockAIService(AIService): class MockAIService(AIService):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -20,8 +21,7 @@ class MockAIService(AIService):
time.sleep(1) time.sleep(1)
return (image_url, image) return (image_url, image)
def run_llm(self, messages, latest_user_message=None, stream = True): def run_llm(self, messages, latest_user_message=None, stream=True):
for i in range(5): for i in range(5):
time.sleep(1) time.sleep(1)
yield({"choices": [{"delta": {"content": f"hello {i}!"}}]}) yield ({"choices": [{"delta": {"content": f"hello {i}!"}}]})

View File

@@ -8,6 +8,7 @@ from pyht.protos.api_pb2 import Format
from services.ai_service import AIService from services.ai_service import AIService
class PlayHTAIService(AIService): class PlayHTAIService(AIService):
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
@@ -23,8 +24,7 @@ class PlayHTAIService(AIService):
voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json", voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json",
sample_rate=16000, sample_rate=16000,
quality="higher", quality="higher",
format=Format.FORMAT_WAV format=Format.FORMAT_WAV)
)
def close(self): def close(self):
super().close() super().close()
@@ -43,14 +43,15 @@ class PlayHTAIService(AIService):
fh = io.BytesIO(b) fh = io.BytesIO(b)
fh.seek(36) fh.seek(36)
(data, size) = struct.unpack('<4sI', fh.read(8)) (data, size) = struct.unpack('<4sI', fh.read(8))
self.logger.info(f"first attempt: data: {data}, size: {hex(size)}, position: {fh.tell()}") self.logger.info(
f"first attempt: data: {data}, size: {hex(size)}, position: {fh.tell()}")
while data != b'data': while data != b'data':
fh.read(size) fh.read(size)
(data, size) = struct.unpack('<4sI', fh.read(8)) (data, size) = struct.unpack('<4sI', fh.read(8))
self.logger.info(f"subsequent data: {data}, size: {hex(size)}, position: {fh.tell()}, data != data: {data != b'data'}") self.logger.info(
f"subsequent data: {data}, size: {hex(size)}, position: {fh.tell()}, data != data: {data != b'data'}")
self.logger.info("position: ", fh.tell()) self.logger.info("position: ", fh.tell())
in_header = False in_header = False
else: else:
if len(chunk): if len(chunk):
yield chunk yield chunk

View File

@@ -6,10 +6,12 @@ from typing import AsyncGenerator, Generator
from dailyai.services.ai_services import AIService from dailyai.services.ai_services import AIService
from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TextQueueFrame from dailyai.queue_frame import EndStreamQueueFrame, QueueFrame, TextQueueFrame
class SimpleAIService(AIService): class SimpleAIService(AIService):
async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]: async def process_frame(self, frame: QueueFrame) -> AsyncGenerator[QueueFrame, None]:
yield frame yield frame
class TestBaseAIService(unittest.IsolatedAsyncioTestCase): class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
async def test_async_input(self): async def test_async_input(self):
service = SimpleAIService() service = SimpleAIService()
@@ -18,6 +20,7 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
TextQueueFrame("hello"), TextQueueFrame("hello"),
EndStreamQueueFrame() EndStreamQueueFrame()
] ]
async def iterate_frames() -> AsyncGenerator[QueueFrame, None]: async def iterate_frames() -> AsyncGenerator[QueueFrame, None]:
for frame in input_frames: for frame in input_frames:
yield frame yield frame

View File

@@ -15,11 +15,12 @@ from dailyai.services.ai_services import AIServiceConfig
from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.deepgram_ai_services import DeepgramTTSService
def add_bot_to_room(room_url, token, expiration) -> None: def add_bot_to_room(room_url, token, expiration) -> None:
# A simple prompt for a simple sample. # A simple prompt for a simple sample.
message_handler = MessageHandler( message_handler = MessageHandler(
""" """
You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's
speech, and your responses will be converted to audio via a TTS service. speech, and your responses will be converted to audio via a TTS service.
Answer user's questions and be friendly, and if you can, give some ideas about how someone Answer user's questions and be friendly, and if you can, give some ideas about how someone
@@ -62,6 +63,7 @@ def add_bot_to_room(room_url, token, expiration) -> None:
services.tts.close() services.tts.close()
services.llm.close() services.llm.close()
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room") parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room")

View File

@@ -20,6 +20,7 @@ from dailyai.message_handler.message_handler import MessageHandler
from dailyai.services.ai_services import AIServiceConfig from dailyai.services.ai_services import AIServiceConfig
from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService from dailyai.services.azure_ai_services import AzureImageGenService, AzureTTSService, AzureLLMService
class StaticSpriteResponse(OrchestratorResponse): class StaticSpriteResponse(OrchestratorResponse):
def __init__( def __init__(
@@ -29,8 +30,8 @@ class StaticSpriteResponse(OrchestratorResponse):
output_queue output_queue
) -> None: ) -> None:
super().__init__(services, message_handler, output_queue) super().__init__(services, message_handler, output_queue)
self.image_bytes:bytes | None = None self.image_bytes: bytes | None = None
self.filenames = None # override this in subclasses self.filenames = None # override this in subclasses
def start_preparation(self) -> None: def start_preparation(self) -> None:
full_path = os.path.join(os.path.dirname(__file__), "sprites/", self.filename) full_path = os.path.join(os.path.dirname(__file__), "sprites/", self.filename)
@@ -82,7 +83,7 @@ def add_bot_to_room(room_url, token, expiration) -> None:
# A simple prompt for a simple sample. # A simple prompt for a simple sample.
message_handler = MessageHandler( message_handler = MessageHandler(
""" """
You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's You are a sample bot in a WebRTC session. You'll receive input as transcriptions of user's
speech, and your responses will be converted to audio via a TTS service. speech, and your responses will be converted to audio via a TTS service.
Answer user's questions and be friendly, and if you can, give some ideas about how someone Answer user's questions and be friendly, and if you can, give some ideas about how someone
@@ -143,6 +144,7 @@ def add_bot_to_room(room_url, token, expiration) -> None:
services.image.close() services.image.close()
services.llm.close() services.llm.close()
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room") parser.add_argument("-u", "--url", type=str, required=True, help="URL of the Daily room")

View File

@@ -4,6 +4,7 @@ import asyncio
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
async def main(room_url): async def main(room_url):
# create a transport service object using environment variables for # create a transport service object using environment variables for
# the transport service's API key, room url, and any other configuration. # the transport service's API key, room url, and any other configuration.

View File

@@ -7,6 +7,7 @@ from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureTTSService from dailyai.services.azure_ai_services import AzureTTSService
from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.deepgram_ai_services import DeepgramTTSService
async def main(room_url): async def main(room_url):
# create a transport service object using environment variables for # create a transport service object using environment variables for
# the transport service's API key, room url, and any other configuration. # the transport service's API key, room url, and any other configuration.
@@ -33,17 +34,20 @@ async def main(room_url):
# Register an event handler so we can play the audio when the participant joins. # Register an event handler so we can play the audio when the participant joins.
print("settting up handler") print("settting up handler")
@transport.event_handler("on_participant_joined") @transport.event_handler("on_participant_joined")
async def on_participant_joined(transport, participant): async def on_participant_joined(transport, participant):
print(f"participant joined: {participant['info']['userName']}") print(f"participant joined: {participant['info']['userName']}")
if participant["info"]["isLocal"]: if participant["info"]["isLocal"]:
return return
audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(f"Hello there, {participant['info']['userName']}!") audio_generator: AsyncGenerator[bytes, None] = tts.run_tts(
f"Hello there, {participant['info']['userName']}!")
async for audio in audio_generator: async for audio in audio_generator:
transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio)) transport.output_queue.put(QueueFrame(FrameType.AUDIO, audio))
print("setting up call state handler") print("setting up call state handler")
@transport.event_handler("on_call_state_updated") @transport.event_handler("on_call_state_updated")
async def on_call_joined(transport, state): async def on_call_joined(transport, state):
print(f"call state callback: {state}") print(f"call state callback: {state}")

View File

@@ -6,6 +6,7 @@ from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
async def main(room_url): async def main(room_url):
meeting_duration_minutes = 1 meeting_duration_minutes = 1
transport = DailyTransportService( transport = DailyTransportService(

View File

@@ -8,6 +8,7 @@ from dailyai.services.open_ai_services import OpenAIImageGenService
local_joined = False local_joined = False
participant_joined = False participant_joined = False
async def main(room_url): async def main(room_url):
meeting_duration_minutes = 1 meeting_duration_minutes = 1
transport = DailyTransportService( transport = DailyTransportService(
@@ -23,8 +24,9 @@ async def main(room_url):
imagegen = OpenAIImageGenService(image_size="1024x1024") imagegen = OpenAIImageGenService(image_size="1024x1024")
image_task = asyncio.create_task( image_task = asyncio.create_task(
imagegen.run_to_queue(transport.send_queue, [TextQueueFrame("a cat in the style of picasso")]) imagegen.run_to_queue(
) transport.send_queue, [
TextQueueFrame("a cat in the style of picasso")]))
@transport.event_handler("on_participant_joined") @transport.event_handler("on_participant_joined")
async def on_participant_joined(transport, participant): async def on_participant_joined(transport, participant):

View File

@@ -7,7 +7,8 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame from dailyai.queue_frame import EndStreamQueueFrame, LLMMessagesQueueFrame
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
async def main(room_url:str):
async def main(room_url: str):
global transport global transport
global llm global llm
global tts global tts

View File

@@ -7,6 +7,7 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
async def main(room_url): async def main(room_url):
meeting_duration_minutes = 5 meeting_duration_minutes = 5
transport = DailyTransportService( transport = DailyTransportService(
@@ -98,7 +99,7 @@ async def main(room_url):
await transport.run() await transport.run()
if __name__=="__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Simple Daily Bot Sample") parser = argparse.ArgumentParser(description="Simple Daily Bot Sample")
parser.add_argument( parser.add_argument(
"-u", "--url", type=str, required=True, help="URL of the Daily room to join" "-u", "--url", type=str, required=True, help="URL of the Daily room to join"

View File

@@ -8,7 +8,8 @@ from dailyai.services.daily_transport_service import DailyTransportService
from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService
from dailyai.queue_aggregators import LLMContextAggregator from dailyai.queue_aggregators import LLMContextAggregator
async def main(room_url:str, token):
async def main(room_url: str, token):
global transport global transport
global llm global llm
global tts global tts

View File

@@ -11,7 +11,7 @@ from dailyai.services.azure_ai_services import AzureLLMService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
async def main(room_url:str, token): async def main(room_url: str, token):
global transport global transport
global llm global llm
global tts global tts

View File

@@ -11,7 +11,8 @@ from dailyai.queue_frame import QueueFrame, FrameType
from dailyai.services.fal_ai_services import FalImageGenService from dailyai.services.fal_ai_services import FalImageGenService
from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService
async def main(room_url:str, token):
async def main(room_url: str, token):
global transport global transport
global llm global llm
global tts global tts
@@ -32,7 +33,6 @@ async def main(room_url:str, token):
tts = AzureTTSService() tts = AzureTTSService()
img = FalImageGenService() img = FalImageGenService()
async def handle_transcriptions(): async def handle_transcriptions():
print("handle_transcriptions got called") print("handle_transcriptions got called")
@@ -41,7 +41,7 @@ async def main(room_url:str, token):
print(f"transcription message: {message}") print(f"transcription message: {message}")
if message["session_id"] == transport.my_participant_id: if message["session_id"] == transport.my_participant_id:
continue continue
finder = message["text"].find("start over") finder = message["text"].find("start over")
print(f"finder: {finder}") print(f"finder: {finder}")
if finder >= 0: if finder >= 0:
async for audio in tts.run_tts(f"Resetting."): async for audio in tts.run_tts(f"Resetting."):
@@ -69,7 +69,8 @@ async def main(room_url:str, token):
if participant["info"]["isLocal"]: if participant["info"]["isLocal"]:
return return
async for audio in tts.run_tts("Describe an image, and I'll create it."): async for audio in tts.run_tts("Describe an image, and I'll create it."):
audio_generator = tts.run_tts(f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.") audio_generator = tts.run_tts(
f"Hello, {participant['info']['userName']}! Describe an image and I'll create it. To start over, just say 'start over'.")
async for audio in audio_generator: async for audio in audio_generator:
transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio)) transport.output_queue.put(QueueFrame(FrameType.AUDIO_FRAME, audio))