diff --git a/pyproject.toml b/pyproject.toml index fa2d6bdcf..d9e155237 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,8 @@ dependencies = [ "google-cloud-texttospeech", "azure-cognitiveservices-speech", "pyht", - "opentelemetry-sdk" + "opentelemetry-sdk", + "aiohttp" ] [tool.setuptools.packages.find] diff --git a/src/dailyai/services/ai_services.py b/src/dailyai/services/ai_services.py index 8bc0b8e52..9c52d43f8 100644 --- a/src/dailyai/services/ai_services.py +++ b/src/dailyai/services/ai_services.py @@ -4,8 +4,6 @@ from abc import abstractmethod from collections.abc import AsyncGenerator from dataclasses import dataclass -from typing import Generator -from PIL import Image class AIService: @@ -20,7 +18,7 @@ class LLMService(AIService): @abstractmethod async def run_llm_async( self, messages - ) -> AsyncGenerator[str, None, None]: + ) -> AsyncGenerator[str, None]: pass # Generate a responses to a prompt. Returns the response @@ -39,14 +37,14 @@ class TTSService(AIService): # Converts the sentence to audio. Yields a list of audio frames that can # be sent to the microphone device @abstractmethod - async def run_tts(self, sentence) -> AsyncGenerator[bytes, None, None]: + async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]: pass class ImageGenService(AIService): # Renders the image. Returns an Image object. @abstractmethod - async def run_image_gen(self, sentence) -> tuple[str, Image.Image]: + async def run_image_gen(self, sentence) -> tuple[str, bytes]: pass diff --git a/src/dailyai/services/azure_ai_services.py b/src/dailyai/services/azure_ai_services.py index 907b33c83..9104b7304 100644 --- a/src/dailyai/services/azure_ai_services.py +++ b/src/dailyai/services/azure_ai_services.py @@ -25,7 +25,7 @@ class AzureTTSService(TTSService): self.speech_config = SpeechConfig(subscription=speech_key, region=speech_region) self.speech_synthesizer = SpeechSynthesizer(speech_config=self.speech_config, audio_config=None) - async def run_tts(self, sentence) -> AsyncGenerator[bytes, None, None]: + async def run_tts(self, sentence) -> AsyncGenerator[bytes, None]: self.logger.info("Running azure tts") ssml = "" \ @@ -99,7 +99,7 @@ class AzureImageGenServiceREST(ImageGenService): self.api_version = api_version or "2023-06-01-preview" self.model = model or os.getenv("AZURE_DALLE_DEPLOYMENT_ID") - async def run_image_gen(self, sentence, size) -> tuple[str, Image.Image]: + async def run_image_gen(self, sentence, size) -> tuple[str, bytes]: # TODO hoist the session to app-level async with aiohttp.ClientSession() as session: url = f"{self.azure_endpoint}openai/images/generations:submit?api-version={self.api_version}" @@ -115,6 +115,7 @@ class AzureImageGenServiceREST(ImageGenService): status = "" attempts_left = 120 + json_response = None while status != "succeeded": attempts_left -= 1 if attempts_left == 0: @@ -125,7 +126,9 @@ class AzureImageGenServiceREST(ImageGenService): json_response = await response.json() status = json_response["status"] - image_url = json_response["result"]["data"][0]["url"] + image_url = json_response["result"]["data"][0]["url"] if json_response else None + if not image_url: + raise Exception("Image generation failed") # Load the image from the url async with session.get(image_url) as response: diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index 92a325d84..025843442 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -23,14 +23,14 @@ class DailyTransportService(EventHandler): def __init__( self, room_url: str, - token: str, + token: str | None, bot_name: str, duration: float = 10, ): super().__init__() self.bot_name: str = bot_name self.room_url: str = room_url - self.token: str = token + self.token: str | None = token self.duration: float = duration self.expiration = time.time() + duration * 60 @@ -38,6 +38,9 @@ class DailyTransportService(EventHandler): self.is_interrupted = Event() self.stop_threads = Event() self.story_started = False + self.mic_enabled = False + self.mic_sample_rate = 16000 + self.camera_enabled = False self.logger: logging.Logger = logging.getLogger("dailyai") @@ -129,6 +132,7 @@ class DailyTransportService(EventHandler): self.configure_daily() self.running_thread = Thread(target=self.run_daily, daemon=True) self.running_thread.start() + self.running_thread.join() def run_daily(self): # TODO: this loop could, I think, be replaced with a timer and an event diff --git a/src/samples/theoretical-to-real/01-say-one-thing.py b/src/samples/theoretical-to-real/01-say-one-thing.py new file mode 100644 index 000000000..514d401b9 --- /dev/null +++ b/src/samples/theoretical-to-real/01-say-one-thing.py @@ -0,0 +1,69 @@ +import asyncio +from typing import AsyncGenerator + +from dailyai.output_queue import OutputQueueFrame, FrameType +from dailyai.services.daily_transport_service import DailyTransportService +from dailyai.services.azure_ai_services import AzureTTSService + +class Sample01Transport(DailyTransportService): + def __init__( + self, + room_url: str, + token: str | None, + bot_name: str, + duration: float = 10, + ): + super().__init__(room_url, token, bot_name, duration) + + def set_audio_generator(self, audio_generator) -> None: + self.audio_generator = audio_generator + + async def play_audio(self): + print("playing audio", self.audio_generator) + async for audio in self.audio_generator: + print("putting frame on queue") + self.output_queue.put(OutputQueueFrame(FrameType.AUDIO_FRAME, audio)) + + def on_participant_joined(self, participant): + super().on_participant_joined(participant) + asyncio.run(self.play_audio()) + +async def main(room_url): + # create a transport service object using environment variables for + # the transport service's API key, room url, and any other configuration. + # services can all define and document the environment variables they use. + # services all also take an optional config object that is used instead of + # environment variables. + # + # the abstract transport service APIs presumably can map pretty closely + # to the daily-python basic API + meeting_duration_minutes = 4 + transport = Sample01Transport( + room_url, + None, + "Say One Thing", + meeting_duration_minutes, + ) + transport.mic_enabled = True + + # similarly, create a tts service + tts = AzureTTSService() + + # ask the transport to create a local audio "device"/queue for + # chunks of audio to play sequentially. the "mic" object is a handle + # we can use to inspect and control the queue if we need to. in this + # case we will pipe into this queue from the tts service + audio_generator: AsyncGenerator[bytes, None] = tts.run_tts("hello world") + + # Should this just happen when we create the object? + transport.set_audio_generator(audio_generator) + try: + transport.run() + except KeyboardInterrupt: + print("Keyboard interrupt") + finally: + transport.stop() + + +if __name__ == "__main__": + asyncio.run(main("https://moishe.daily.co/Lettvins")) diff --git a/src/samples/05-sync-speech-and-text.py b/src/samples/theoretical-to-real/05-sync-speech-and-text.py similarity index 100% rename from src/samples/05-sync-speech-and-text.py rename to src/samples/theoretical-to-real/05-sync-speech-and-text.py