From d27122e35e7815757fa5fa8d57e22ef076bb1ca7 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Fri, 9 Feb 2024 09:10:28 -0600 Subject: [PATCH 01/11] Create LICENSE --- LICENSE | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..cd6220df2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +BSD 2-Clause License + +Copyright (c) 2024, Daily + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. From 444418d94c2e1483118b3a7bb4f9904f211b34ac Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Fri, 9 Feb 2024 10:26:39 -0500 Subject: [PATCH 02/11] Make the README okay-enough for limited public release --- README.md | 35 +---------------------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/README.md b/README.md index c1122778b..bc585fa3a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Build conversational, multi-modal AI apps with real-time voice and video, like this: -_Demo Video_ +_Demo Video to come_ With built-in support for many of the best AI platforms (or [add your own](/docs)): @@ -15,22 +15,6 @@ With built-in support for many of the best AI platforms (or [add your own](/docs ## Step 1: Get Started -Installation here. Also sign up for a Daily account, I guess? also we need an ENV - -Requires python 3.11 or later. Don't forget virtualenv - -pip install vs download and build? - -## Step 2: Build Things - -Once you've got the SDK working, head over to the [docs folder](/docs) to start building! - ---- - -# Old Readme - -This SDK can help you build applications that participate in WebRTC meetings and use various AI services to interact with other participants. - ## Build/Install _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_ @@ -66,23 +50,6 @@ Tou can run the simple sample like so: ``` python src/examples/theoretical-to-real/01-say-one-thing.py -u -k ``` - -Note that the sample uses Azure's TTS and LLM services. You'll need to set the following environment variables for the sample to work: - -``` -AZURE_SPEECH_SERVICE_KEY -AZURE_SPEECH_SERVICE_REGION -AZURE_CHATGPT_KEY -AZURE_CHATGPT_ENDPOINT -AZURE_CHATGPT_DEPLOYMENT_ID -``` - -If you have those environment variables stored in an .env file, you can quickly load them into your terminal's environment by running this: - -```bash -export $(grep -v '^#' .env | xargs) -``` - ## Overview The Daily AI SDK allows you to build applications that can participate in WebRTC sessions and interact with AI Services. Some examples of what you can build with this: From 0e0c992f5940e9df683ab6fd83342bded2cf33b4 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Sat, 10 Feb 2024 09:22:52 -0500 Subject: [PATCH 03/11] Ollama LLM service --- src/dailyai/services/ollama_ai_services.py | 42 ++++++++++++++++++++++ src/dailyai/services/open_ai_services.py | 5 +-- 2 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 src/dailyai/services/ollama_ai_services.py diff --git a/src/dailyai/services/ollama_ai_services.py b/src/dailyai/services/ollama_ai_services.py new file mode 100644 index 000000000..c065e0b7b --- /dev/null +++ b/src/dailyai/services/ollama_ai_services.py @@ -0,0 +1,42 @@ +from openai import AsyncOpenAI + +import json +from collections.abc import AsyncGenerator + +from dailyai.services.ai_services import LLMService + + +class OLLamaLLMService(LLMService): + def __init__(self, model="llama2", base_url='http://localhost:11434/v1'): + super().__init__() + self._model = model + self._client = AsyncOpenAI(api_key="ollama", base_url=base_url) + + async def get_response(self, messages, stream): + return await self._client.chat.completions.create( + stream=stream, + messages=messages, + model=self._model + ) + + async def run_llm_async(self, messages) -> AsyncGenerator[str, None]: + messages_for_log = json.dumps(messages) + self.logger.debug(f"Generating chat via openai: {messages_for_log}") + + chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages) + async for chunk in chunks: + if len(chunk.choices) == 0: + continue + + if chunk.choices[0].delta.content: + yield chunk.choices[0].delta.content + + async def run_llm(self, messages) -> str | None: + messages_for_log = json.dumps(messages) + self.logger.debug(f"Generating chat via openai: {messages_for_log}") + + response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages) + if response and len(response.choices) > 0: + return response.choices[0].message.content + else: + return None diff --git a/src/dailyai/services/open_ai_services.py b/src/dailyai/services/open_ai_services.py index 7892af06b..2aea60fa9 100644 --- a/src/dailyai/services/open_ai_services.py +++ b/src/dailyai/services/open_ai_services.py @@ -1,15 +1,12 @@ -import requests import aiohttp -import asyncio from PIL import Image import io from openai import AsyncOpenAI -import os import json from collections.abc import AsyncGenerator -from dailyai.services.ai_services import AIService, TTSService, LLMService, ImageGenService +from dailyai.services.ai_services import LLMService, ImageGenService class OpenAILLMService(LLMService): From 815aa2bc3e2ed45d3eb2509b9b9ea203e628614e Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Sat, 10 Feb 2024 09:29:08 -0500 Subject: [PATCH 04/11] Another autopep8 formatting pass --- src/dailyai/queue_aggregators.py | 14 ++++++---- src/dailyai/queue_frame.py | 2 ++ .../services/base_transport_service.py | 1 + .../services/daily_transport_service.py | 2 +- src/dailyai/services/fal_ai_services.py | 8 +++++- .../services/local_transport_service.py | 8 ++++-- src/examples/foundational/01-say-one-thing.py | 6 ++++- .../foundational/02-llm-say-one-thing.py | 13 ++++++--- src/examples/foundational/03-still-frame.py | 6 ++++- .../foundational/04-utterance-and-speech.py | 15 ++++++++--- .../foundational/05-sync-speech-and-image.py | 17 +++++++++--- .../foundational/06-listen-and-respond.py | 12 ++++++--- src/examples/foundational/06a-image-sync.py | 16 ++++++++--- src/examples/foundational/07-interruptible.py | 10 +++++-- src/examples/foundational/08-bots-arguing.py | 27 ++++++++++++++----- src/examples/foundational/10-wake-word.py | 12 ++++++--- src/examples/foundational/11-sound-effects.py | 18 ++++++++----- .../foundational/13-whisper-transcription.py | 1 + .../foundational/13a-whisper-local.py | 2 +- src/examples/foundational/support/runner.py | 9 ++++--- src/examples/internal/11a-dial-out.py | 5 ++-- 21 files changed, 152 insertions(+), 52 deletions(-) diff --git a/src/dailyai/queue_aggregators.py b/src/dailyai/queue_aggregators.py index 80c99bc19..182d7d479 100644 --- a/src/dailyai/queue_aggregators.py +++ b/src/dailyai/queue_aggregators.py @@ -61,13 +61,17 @@ class LLMContextAggregator(AIService): # TODO: split up transcription by participant if self.complete_sentences: - self.sentence += frame.text # type: ignore -- the linter thinks this isn't a TextQueueFrame, even though we check it above + # type: ignore -- the linter thinks this isn't a TextQueueFrame, even + # though we check it above + self.sentence += frame.text if self.sentence.endswith((".", "?", "!")): self.messages.append({"role": self.role, "content": self.sentence}) self.sentence = "" yield LLMMessagesQueueFrame(self.messages) else: - self.messages.append({"role": self.role, "content": frame.text}) # type: ignore -- the linter thinks this isn't a TextQueueFrame, even though we check it above + # type: ignore -- the linter thinks this isn't a TextQueueFrame, even + # though we check it above + self.messages.append({"role": self.role, "content": frame.text}) yield LLMMessagesQueueFrame(self.messages) async def finalize(self) -> AsyncGenerator[QueueFrame, None]: @@ -79,9 +83,9 @@ class LLMContextAggregator(AIService): class LLMUserContextAggregator(LLMContextAggregator): def __init__(self, - messages: list[dict], - bot_participant_id=None, - complete_sentences=True): + messages: list[dict], + bot_participant_id=None, + complete_sentences=True): super().__init__(messages, "user", bot_participant_id, complete_sentences, pass_through=False) diff --git a/src/dailyai/queue_frame.py b/src/dailyai/queue_frame.py index 3e249b38c..d43dbdf82 100644 --- a/src/dailyai/queue_frame.py +++ b/src/dailyai/queue_frame.py @@ -18,9 +18,11 @@ class StartStreamQueueFrame(ControlQueueFrame): class EndStreamQueueFrame(ControlQueueFrame): pass + class LLMResponseEndQueueFrame(QueueFrame): pass + @dataclass() class AudioQueueFrame(QueueFrame): data: bytes diff --git a/src/dailyai/services/base_transport_service.py b/src/dailyai/services/base_transport_service.py index 139edc6ff..b05e7e373 100644 --- a/src/dailyai/services/base_transport_service.py +++ b/src/dailyai/services/base_transport_service.py @@ -16,6 +16,7 @@ from dailyai.queue_frame import ( StartStreamQueueFrame, ) + class BaseTransportService(): def __init__( diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index 575a6560b..2f07e6d63 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -45,7 +45,7 @@ class DailyTransportService(BaseTransportService, EventHandler): start_transcription: bool = False, **kwargs, ): - super().__init__(**kwargs) # This will call BaseTransportService.__init__ method, not EventHandler + super().__init__(**kwargs) # This will call BaseTransportService.__init__ method, not EventHandler self._room_url: str = room_url self._bot_name: str = bot_name diff --git a/src/dailyai/services/fal_ai_services.py b/src/dailyai/services/fal_ai_services.py index ca1f93d89..9464b46dd 100644 --- a/src/dailyai/services/fal_ai_services.py +++ b/src/dailyai/services/fal_ai_services.py @@ -13,7 +13,13 @@ from dailyai.services.ai_services import ImageGenService class FalImageGenService(ImageGenService): - def __init__(self, *, image_size, aiohttp_session: aiohttp.ClientSession, key_id=None, key_secret=None): + def __init__( + self, + *, + image_size, + aiohttp_session: aiohttp.ClientSession, + key_id=None, + key_secret=None): super().__init__(image_size) self._aiohttp_session = aiohttp_session if key_id: diff --git a/src/dailyai/services/local_transport_service.py b/src/dailyai/services/local_transport_service.py index 91af6763c..7538d3b06 100644 --- a/src/dailyai/services/local_transport_service.py +++ b/src/dailyai/services/local_transport_service.py @@ -22,11 +22,15 @@ class LocalTransportService(BaseTransportService): async def _write_frame_to_tkinter(self, frame: bytes): data = f"P6 {self._camera_width} {self._camera_height} 255 ".encode() + frame - photo = tk.PhotoImage(width=self._camera_width, height=self._camera_height, data=data, format="PPM") + photo = tk.PhotoImage( + width=self._camera_width, + height=self._camera_height, + data=data, + format="PPM") self._image_label.config(image=photo) # This holds a reference to the photo, preventing it from being garbage collected. - self._image_label.image = photo # type: ignore + self._image_label.image = photo # type: ignore def write_frame_to_camera(self, frame: bytes): if self._camera_enabled and self._loop: diff --git a/src/examples/foundational/01-say-one-thing.py b/src/examples/foundational/01-say-one-thing.py index 989da84af..37136facf 100644 --- a/src/examples/foundational/01-say-one-thing.py +++ b/src/examples/foundational/01-say-one-thing.py @@ -7,6 +7,7 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from examples.foundational.support.runner import configure + async def main(room_url): async with aiohttp.ClientSession() as session: # create a transport service object using environment variables for @@ -25,7 +26,10 @@ async def main(room_url): meeting_duration_minutes, mic_enabled=True ) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID")) # Register an event handler so we can play the audio when the participant joins. @transport.event_handler("on_participant_joined") diff --git a/src/examples/foundational/02-llm-say-one-thing.py b/src/examples/foundational/02-llm-say-one-thing.py index 7a30a906f..b15023380 100644 --- a/src/examples/foundational/02-llm-say-one-thing.py +++ b/src/examples/foundational/02-llm-say-one-thing.py @@ -11,6 +11,7 @@ from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.open_ai_services import OpenAILLMService from examples.foundational.support.runner import configure + async def main(room_url): async with aiohttp.ClientSession() as session: meeting_duration_minutes = 1 @@ -22,12 +23,18 @@ async def main(room_url): mic_enabled=True ) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID")) # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) # tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE")) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - #llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + # llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY")) messages = [{ "role": "system", "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world." diff --git a/src/examples/foundational/03-still-frame.py b/src/examples/foundational/03-still-frame.py index 06d40448b..de368d5d6 100644 --- a/src/examples/foundational/03-still-frame.py +++ b/src/examples/foundational/03-still-frame.py @@ -28,7 +28,11 @@ async def main(room_url): camera_height=1024 ) - imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + imagegen = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) # imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024") # imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL")) diff --git a/src/examples/foundational/04-utterance-and-speech.py b/src/examples/foundational/04-utterance-and-speech.py index 18e7c2ff7..17bd32797 100644 --- a/src/examples/foundational/04-utterance-and-speech.py +++ b/src/examples/foundational/04-utterance-and-speech.py @@ -10,6 +10,7 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from examples.foundational.support.runner import configure + async def main(room_url: str): async with aiohttp.ClientSession() as session: transport = DailyTransportService( @@ -22,9 +23,17 @@ async def main(room_url: str): camera_enabled=False ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - azure_tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - elevenlabs_tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + azure_tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) + elevenlabs_tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID")) messages = [{"role": "system", "content": "tell the user a joke about llamas"}] diff --git a/src/examples/foundational/05-sync-speech-and-image.py b/src/examples/foundational/05-sync-speech-and-image.py index 55e4ec9f7..dfef2bc15 100644 --- a/src/examples/foundational/05-sync-speech-and-image.py +++ b/src/examples/foundational/05-sync-speech-and-image.py @@ -11,6 +11,7 @@ from dailyai.services.open_ai_services import OpenAIImageGenService from examples.foundational.support.runner import configure + async def main(room_url): async with aiohttp.ClientSession() as session: meeting_duration_minutes = 5 @@ -26,11 +27,21 @@ async def main(room_url): camera_height=1024 ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV") + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="ErXwobaYiN019PkySvjV") # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + dalle = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) # dalle = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024") # dalle = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL")) diff --git a/src/examples/foundational/06-listen-and-respond.py b/src/examples/foundational/06-listen-and-respond.py index df8895731..fa5e077cc 100644 --- a/src/examples/foundational/06-listen-and-respond.py +++ b/src/examples/foundational/06-listen-and-respond.py @@ -6,6 +6,7 @@ from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator from examples.foundational.support.runner import configure + async def main(room_url: str, token): transport = DailyTransportService( room_url, @@ -15,11 +16,16 @@ async def main(room_url: str, token): start_transcription=True, mic_enabled=True, mic_sample_rate=16000, - camera_enabled = False + camera_enabled=False ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): diff --git a/src/examples/foundational/06a-image-sync.py b/src/examples/foundational/06a-image-sync.py index fbef24364..032d828c1 100644 --- a/src/examples/foundational/06a-image-sync.py +++ b/src/examples/foundational/06a-image-sync.py @@ -18,6 +18,7 @@ from dailyai.services.fal_ai_services import FalImageGenService from examples.foundational.support.runner import configure + class ImageSyncAggregator(AIService): def __init__(self, speaking_path: str, waiting_path: str): self._speaking_image = Image.open(speaking_path) @@ -46,9 +47,18 @@ async def main(room_url: str, token): transport._mic_enabled = True transport._mic_sample_rate = 16000 - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - img = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) + img = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) async def get_images(): get_speaking_task = asyncio.create_task( diff --git a/src/examples/foundational/07-interruptible.py b/src/examples/foundational/07-interruptible.py index cd7f9de99..6cae19f5c 100644 --- a/src/examples/foundational/07-interruptible.py +++ b/src/examples/foundational/07-interruptible.py @@ -10,6 +10,7 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from examples.foundational.support.runner import configure + async def main(room_url: str, token): async with aiohttp.ClientSession() as session: transport = DailyTransportService( @@ -23,8 +24,13 @@ async def main(room_url: str, token): camera_enabled=False, ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) async def run_response(user_speech, tma_in, tma_out): await tts.run_to_queue( diff --git a/src/examples/foundational/08-bots-arguing.py b/src/examples/foundational/08-bots-arguing.py index 95fc62e40..a5329086e 100644 --- a/src/examples/foundational/08-bots-arguing.py +++ b/src/examples/foundational/08-bots-arguing.py @@ -10,7 +10,8 @@ from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame from examples.foundational.support.runner import configure -async def main(room_url:str): + +async def main(room_url: str): async with aiohttp.ClientSession() as session: transport = DailyTransportService( room_url, @@ -24,16 +25,30 @@ async def main(room_url:str): camera_height=1024 ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts1 = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - tts2 = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl") - dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts1 = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) + tts2 = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="jBpfuIE2acCO8z3wKNLl") + dalle = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) bot1_messages = [ {"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."}, ] bot2_messages = [ - {"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, + { + "role": "system", + "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, ] async def get_bot1_statement(): diff --git a/src/examples/foundational/10-wake-word.py b/src/examples/foundational/10-wake-word.py index ccf706d50..54db2468c 100644 --- a/src/examples/foundational/10-wake-word.py +++ b/src/examples/foundational/10-wake-word.py @@ -71,7 +71,7 @@ class TranscriptFilter(AIService): class NameCheckFilter(AIService): - def __init__(self, names:list[str]): + def __init__(self, names: list[str]): self.names = names self.sentence = "" @@ -123,8 +123,14 @@ async def main(room_url: str, token): transport._camera_width = 720 transport._camera_height = 1280 - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl") + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="jBpfuIE2acCO8z3wKNLl") isa = ImageSyncAggregator() @transport.event_handler("on_first_other_participant_joined") diff --git a/src/examples/foundational/11-sound-effects.py b/src/examples/foundational/11-sound-effects.py index 743120f23..954f363d7 100644 --- a/src/examples/foundational/11-sound-effects.py +++ b/src/examples/foundational/11-sound-effects.py @@ -14,7 +14,7 @@ from typing import AsyncGenerator from examples.foundational.support.runner import configure -logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever +logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever logger = logging.getLogger("dailyai") logger.setLevel(logging.DEBUG) @@ -36,8 +36,6 @@ for file in sound_files: sounds[file] = audio_file.readframes(-1) - - class OutboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -50,6 +48,7 @@ class OutboundSoundEffectWrapper(AIService): else: yield frame + class InboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -75,14 +74,20 @@ async def main(room_url: str, token): camera_enabled=False ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV") - + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="ErXwobaYiN019PkySvjV") @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): await tts.say("Hi, I'm listening!", transport.send_queue) await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) + async def handle_transcriptions(): messages = [ {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, @@ -117,7 +122,6 @@ async def main(room_url: str, token): ) ) - transport.transcription_settings["extra"]["punctuate"] = True await asyncio.gather(transport.run(), handle_transcriptions()) diff --git a/src/examples/foundational/13-whisper-transcription.py b/src/examples/foundational/13-whisper-transcription.py index 147b20e22..726616c5f 100644 --- a/src/examples/foundational/13-whisper-transcription.py +++ b/src/examples/foundational/13-whisper-transcription.py @@ -5,6 +5,7 @@ from dailyai.services.whisper_ai_services import WhisperSTTService from examples.foundational.support.runner import configure + async def main(room_url: str): transport = DailyTransportService( room_url, diff --git a/src/examples/foundational/13a-whisper-local.py b/src/examples/foundational/13a-whisper-local.py index 0a30ad721..6c764c1a9 100644 --- a/src/examples/foundational/13a-whisper-local.py +++ b/src/examples/foundational/13a-whisper-local.py @@ -17,7 +17,7 @@ async def main(room_url: str): camera_enabled=False, speaker_enabled=True, duration_minutes=meeting_duration_minutes, - start_transcription = True + start_transcription=True ) stt = WhisperSTTService() transcription_output_queue = asyncio.Queue() diff --git a/src/examples/foundational/support/runner.py b/src/examples/foundational/support/runner.py index 94263c0a6..b4b7d4862 100644 --- a/src/examples/foundational/support/runner.py +++ b/src/examples/foundational/support/runner.py @@ -7,6 +7,7 @@ import requests from dotenv import load_dotenv load_dotenv() + def configure(): parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") parser.add_argument( @@ -26,11 +27,11 @@ def configure(): key = args.apikey or os.getenv("DAILY_API_KEY") if not url: - raise Exception("No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.") - + raise Exception( + "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.") + if not key: raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.") - # Create a meeting token for the given room with an expiration 1 hour in the future. room_name: str = urllib.parse.urlparse(url).path[1:] @@ -49,4 +50,4 @@ def configure(): token: str = res.json()["token"] - return (url, token) \ No newline at end of file + return (url, token) diff --git a/src/examples/internal/11a-dial-out.py b/src/examples/internal/11a-dial-out.py index 9d38299f4..b16c72ed1 100644 --- a/src/examples/internal/11a-dial-out.py +++ b/src/examples/internal/11a-dial-out.py @@ -30,8 +30,6 @@ for file in sound_files: sounds[file] = audio_file.readframes(-1) - - class OutboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -44,6 +42,7 @@ class OutboundSoundEffectWrapper(AIService): else: yield frame + class InboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -81,6 +80,7 @@ async def main(room_url: str, token, phone): async def on_first_other_participant_joined(transport): await tts.say("Hi, I'm listening!", transport.send_queue) await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) + async def handle_transcriptions(): messages = [ {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, @@ -124,7 +124,6 @@ async def main(room_url: str, token, phone): transport.start_recording() transport.dialout(phone) - transport.transcription_settings["extra"]["punctuate"] = True await asyncio.gather(transport.run(), handle_transcriptions()) From 4fecc10808cf3d5c49165f4db476cfefb611db15 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Tue, 13 Feb 2024 14:17:09 -0500 Subject: [PATCH 05/11] Call client.leave on keyboard interrupt --- src/dailyai/services/base_transport_service.py | 6 ++++++ src/dailyai/services/daily_transport_service.py | 12 ++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/dailyai/services/base_transport_service.py b/src/dailyai/services/base_transport_service.py index b05e7e373..beaab2b20 100644 --- a/src/dailyai/services/base_transport_service.py +++ b/src/dailyai/services/base_transport_service.py @@ -76,6 +76,9 @@ class BaseTransportService(): except Exception as e: self._logger.error(f"Exception {e}") raise e + finally: + # Do anything that must be done to clean up + self._post_run() self._stop_threads.set() @@ -87,6 +90,9 @@ class BaseTransportService(): if self._speaker_enabled: self._receive_audio_thread.join() + def _post_run(self): + pass + def stop(self): self._stop_threads.set() diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index 2f07e6d63..f29633d87 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -1,8 +1,8 @@ import asyncio import inspect import logging +import signal import threading -import time import types from functools import partial @@ -11,7 +11,7 @@ from dailyai.queue_frame import ( TranscriptionQueueFrame, ) -from threading import Thread, Event +from threading import Event from daily import ( EventHandler, @@ -194,6 +194,14 @@ class DailyTransportService(BaseTransportService, EventHandler): if self._token and self._start_transcription: self.client.start_transcription(self.transcription_settings) + signal.signal(signal.SIGINT, self.process_interrupt_handler) + + def process_interrupt_handler(self, signum, frame): + self._post_run() + + def _post_run(self): + self.client.leave() + def on_first_other_participant_joined(self): pass From 1992b7e79e0714a4951133c3a84c5c2c44212723 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Wed, 14 Feb 2024 12:10:47 -0500 Subject: [PATCH 06/11] fix sigint handling --- src/dailyai/services/base_transport_service.py | 2 ++ src/dailyai/services/daily_transport_service.py | 3 +++ src/dailyai/services/{to_be_updated => }/playht_ai_service.py | 0 3 files changed, 5 insertions(+) rename src/dailyai/services/{to_be_updated => }/playht_ai_service.py (100%) diff --git a/src/dailyai/services/base_transport_service.py b/src/dailyai/services/base_transport_service.py index beaab2b20..530990cfa 100644 --- a/src/dailyai/services/base_transport_service.py +++ b/src/dailyai/services/base_transport_service.py @@ -91,6 +91,8 @@ class BaseTransportService(): self._receive_audio_thread.join() def _post_run(self): + # Note that this function must be idempotent! It can be called multiple times + # if, for example, a keyboard interrupt occurs. pass def stop(self): diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index f29633d87..44c2f63fb 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -194,10 +194,13 @@ class DailyTransportService(BaseTransportService, EventHandler): if self._token and self._start_transcription: self.client.start_transcription(self.transcription_settings) + self.original_sigint_handler = signal.getsignal(signal.SIGINT) signal.signal(signal.SIGINT, self.process_interrupt_handler) def process_interrupt_handler(self, signum, frame): self._post_run() + if callable(self.original_sigint_handler): + self.original_sigint_handler(signum, frame) def _post_run(self): self.client.leave() diff --git a/src/dailyai/services/to_be_updated/playht_ai_service.py b/src/dailyai/services/playht_ai_service.py similarity index 100% rename from src/dailyai/services/to_be_updated/playht_ai_service.py rename to src/dailyai/services/playht_ai_service.py From 97a4cb8b7f07d9b202e80af29e45527e8fc2d185 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Wed, 14 Feb 2024 12:16:48 -0500 Subject: [PATCH 07/11] Update playht tts service --- src/dailyai/services/playht_ai_service.py | 26 +++++++++++-------- src/examples/foundational/01-say-one-thing.py | 8 ++++++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/dailyai/services/playht_ai_service.py b/src/dailyai/services/playht_ai_service.py index 4ba9ddc86..ed09f8679 100644 --- a/src/dailyai/services/playht_ai_service.py +++ b/src/dailyai/services/playht_ai_service.py @@ -1,36 +1,40 @@ import io -import os import struct from pyht import Client -from dotenv import load_dotenv from pyht.client import TTSOptions from pyht.protos.api_pb2 import Format -from services.ai_service import AIService +from dailyai.services.ai_services import TTSService -class PlayHTAIService(AIService): - def __init__(self, **kwargs): - super().__init__(**kwargs) +class PlayHTAIService(TTSService): - self.speech_key = os.getenv("PLAY_HT_KEY") or '' - self.user_id = os.getenv("PLAY_HT_USER_ID") or '' + def __init__( + self, + *, + api_key, + user_id, + voice_url + ): + super().__init__() + + self.speech_key = api_key + self.user_id = user_id self.client = Client( user_id=self.user_id, api_key=self.speech_key, ) self.options = TTSOptions( - voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json", + voice=voice_url, sample_rate=16000, quality="higher", format=Format.FORMAT_WAV) def close(self): - super().close() self.client.close() - def run_tts(self, sentence): + async def run_tts(self, sentence): b = bytearray() in_header = True for chunk in self.client.tts(sentence, self.options): diff --git a/src/examples/foundational/01-say-one-thing.py b/src/examples/foundational/01-say-one-thing.py index 37136facf..cb81f023e 100644 --- a/src/examples/foundational/01-say-one-thing.py +++ b/src/examples/foundational/01-say-one-thing.py @@ -4,6 +4,7 @@ import os from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from dailyai.services.playht_ai_service import PlayHTAIService from examples.foundational.support.runner import configure @@ -26,10 +27,17 @@ async def main(room_url): meeting_duration_minutes, mic_enabled=True ) + """ tts = ElevenLabsTTSService( aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + """ + tts = PlayHTAIService( + api_key=os.getenv("PLAY_HT_API_KEY"), + user_id=os.getenv("PLAY_HT_USER_ID"), + voice_url=os.getenv("PLAY_HT_VOICE_URL"), + ) # Register an event handler so we can play the audio when the participant joins. @transport.event_handler("on_participant_joined") From dcbd79333abef1abc896bf8798874ecfd4014995 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Wed, 14 Feb 2024 12:53:20 -0500 Subject: [PATCH 08/11] make destructor call client.close in PlayHT service --- src/dailyai/services/playht_ai_service.py | 2 +- src/examples/foundational/01-say-one-thing.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/dailyai/services/playht_ai_service.py b/src/dailyai/services/playht_ai_service.py index ed09f8679..350e5cb88 100644 --- a/src/dailyai/services/playht_ai_service.py +++ b/src/dailyai/services/playht_ai_service.py @@ -31,7 +31,7 @@ class PlayHTAIService(TTSService): quality="higher", format=Format.FORMAT_WAV) - def close(self): + def __del__(self): self.client.close() async def run_tts(self, sentence): diff --git a/src/examples/foundational/01-say-one-thing.py b/src/examples/foundational/01-say-one-thing.py index cb81f023e..9df54d708 100644 --- a/src/examples/foundational/01-say-one-thing.py +++ b/src/examples/foundational/01-say-one-thing.py @@ -27,6 +27,7 @@ async def main(room_url): meeting_duration_minutes, mic_enabled=True ) + """ tts = ElevenLabsTTSService( aiohttp_session=session, @@ -42,6 +43,7 @@ async def main(room_url): # Register an event handler so we can play the audio when the participant joins. @transport.event_handler("on_participant_joined") async def on_participant_joined(transport, participant): + nonlocal tts if participant["info"]["isLocal"]: return @@ -54,6 +56,7 @@ async def main(room_url): await transport.stop_when_done() await transport.run() + del(tts) if __name__ == "__main__": From 92ec5641d42acfacb2816473d7658661b6e320dc Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Wed, 14 Feb 2024 13:44:59 -0500 Subject: [PATCH 09/11] update deepgram tts to new service structure --- src/dailyai/services/deepgram_ai_service.py | 36 +++++++++++++++++++ .../to_be_updated/deepgram_ai_service.py | 29 --------------- 2 files changed, 36 insertions(+), 29 deletions(-) create mode 100644 src/dailyai/services/deepgram_ai_service.py delete mode 100644 src/dailyai/services/to_be_updated/deepgram_ai_service.py diff --git a/src/dailyai/services/deepgram_ai_service.py b/src/dailyai/services/deepgram_ai_service.py new file mode 100644 index 000000000..1ede1c35d --- /dev/null +++ b/src/dailyai/services/deepgram_ai_service.py @@ -0,0 +1,36 @@ +import os +import aiohttp +import requests + +from dailyai.services.ai_services import TTSService + + +class DeepgramAIService(TTSService): + def __init__( + self, + *, + aiohttp_session: aiohttp.ClientSession, + api_key, + voice, + sample_rate=16000 + ): + super().__init__() + + self._api_key = api_key + self._voice = voice + self._sample_rate = sample_rate + self._aiohttp_session = aiohttp_session + + async def run_tts(self, sentence): + self.logger.info(f"Running deepgram tts for {sentence}") + base_url = "https://api.beta.deepgram.com/v1/speak" + request_url = f"{base_url}?model={self._voice}&encoding=linear16&container=none&sample_rate={self._sample_rate}" + headers = {"authorization": f"token {self._api_key}", "Content-Type": "application/json"} + data = {"text": sentence} + + async with self._aiohttp_session.post( + request_url, headers=headers, json=data + ) as r: + async for chunk in r.content: + if chunk: + yield chunk diff --git a/src/dailyai/services/to_be_updated/deepgram_ai_service.py b/src/dailyai/services/to_be_updated/deepgram_ai_service.py deleted file mode 100644 index b4569e6cd..000000000 --- a/src/dailyai/services/to_be_updated/deepgram_ai_service.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import requests - -from services.ai_service import AIService -from PIL import Image - - -class DeepgramAIService(AIService): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - self.api_key = os.getenv("DEEPGRAM_API_KEY") - - def get_mic_sample_rate(self): - return 24000 - - def run_tts(self, sentence): - self.logger.info(f"Running deepgram tts for {sentence}") - base_url = "https://api.beta.deepgram.com/v1/speak" - # move this to an environment variable - voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1" - request_url = f"{base_url}?model={voice}&encoding=linear16&container=none" - headers = {"authorization": f"token {self.api_key}"} - - r = requests.post(request_url, headers=headers, data=sentence) - self.logger.info( - f"audio fetch status code: {r.status_code}, content length: {len(r.content)}" - ) - yield r.content From 20091d91c97699c01802ef28e0ff649b24e866a4 Mon Sep 17 00:00:00 2001 From: Moishe Lettvin Date: Tue, 27 Feb 2024 13:09:55 -0500 Subject: [PATCH 10/11] cleanup client properties and unsubscribe from camera --- .../services/daily_transport_service.py | 82 ++++++++++--------- 1 file changed, 44 insertions(+), 38 deletions(-) diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index 44c2f63fb..c4266406d 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -149,47 +149,53 @@ class DailyTransportService(BaseTransportService, EventHandler): Daily.select_speaker_device("speaker") self.client.set_user_name(self._bot_name) - self.client.join(self._room_url, self._token, completion=self.call_joined) + self.client.join( + self._room_url, + self._token, + completion=self.call_joined, + client_settings={ + "inputs": { + "camera": { + "isEnabled": True, + "settings": { + "deviceId": "camera", + }, + }, + "microphone": { + "isEnabled": True, + "settings": { + "deviceId": "mic", + "customConstraints": { + "autoGainControl": {"exact": False}, + "echoCancellation": {"exact": False}, + "noiseSuppression": {"exact": False}, + }, + }, + }, + }, + "publishing": { + "camera": { + "sendSettings": { + "maxQuality": "low", + "encodings": { + "low": { + "maxBitrate": 250000, + "scaleResolutionDownBy": 1.333, + "maxFramerate": 8, + } + }, + } + } + }, + }, + ) self._my_participant_id = self.client.participants()["local"]["id"] - self.client.update_inputs( - { - "camera": { - "isEnabled": True, - "settings": { - "deviceId": "camera", - }, - }, - "microphone": { - "isEnabled": True, - "settings": { - "deviceId": "mic", - "customConstraints": { - "autoGainControl": {"exact": False}, - "echoCancellation": {"exact": False}, - "noiseSuppression": {"exact": False}, - }, - }, - }, + self.client.update_subscription_profiles({ + "base": { + "camera": "unsubscribed", } - ) - - self.client.update_publishing( - { - "camera": { - "sendSettings": { - "maxQuality": "low", - "encodings": { - "low": { - "maxBitrate": 250000, - "scaleResolutionDownBy": 1.333, - "maxFramerate": 8, - } - }, - } - } - } - ) + }) if self._token and self._start_transcription: self.client.start_transcription(self.transcription_settings) From d90fdb1caed05392e93fee2d0f5ee2f3164f2955 Mon Sep 17 00:00:00 2001 From: chadbailey59 Date: Wed, 28 Feb 2024 15:16:44 -0600 Subject: [PATCH 11/11] Isolated changes to add VAD (#32) * added VAD * added separate 'vad enabled' property --- pyproject.toml | 3 + src/dailyai/queue_frame.py | 6 + .../services/base_transport_service.py | 132 +++++++++++++++++- .../services/daily_transport_service.py | 3 +- .../foundational/06-listen-and-respond.py | 23 ++- 5 files changed, 157 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7603f8500..e6aa7e5a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,13 @@ dependencies = [ "fal", "faster_whisper", "google-cloud-texttospeech", + "numpy", "openai", "Pillow", "pyht", "python-dotenv", + "torch", + "pyaudio", "typing-extensions" ] diff --git a/src/dailyai/queue_frame.py b/src/dailyai/queue_frame.py index d43dbdf82..dc111dcbe 100644 --- a/src/dailyai/queue_frame.py +++ b/src/dailyai/queue_frame.py @@ -58,3 +58,9 @@ class LLMMessagesQueueFrame(QueueFrame): class AppMessageQueueFrame(QueueFrame): message: Any participantId: str + +class UserStartedSpeakingFrame(QueueFrame): + pass + +class UserStoppedSpeakingFrame(QueueFrame): + pass \ No newline at end of file diff --git a/src/dailyai/services/base_transport_service.py b/src/dailyai/services/base_transport_service.py index 530990cfa..9ee60f4bb 100644 --- a/src/dailyai/services/base_transport_service.py +++ b/src/dailyai/services/base_transport_service.py @@ -2,10 +2,15 @@ from abc import abstractmethod import asyncio import itertools import logging +import numpy as np +import pyaudio +import torch +import torchaudio import queue import threading import time from typing import AsyncGenerator +from enum import Enum from dailyai.queue_frame import ( AudioQueueFrame, @@ -14,8 +19,57 @@ from dailyai.queue_frame import ( QueueFrame, SpriteQueueFrame, StartStreamQueueFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame ) +torch.set_num_threads(1) + +model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', + model='silero_vad', + force_reload=False) + +(get_speech_timestamps, + save_audio, + read_audio, + VADIterator, + collect_chunks) = utils + +# Taken from utils_vad.py + + +def validate(model, + inputs: torch.Tensor): + with torch.no_grad(): + outs = model(inputs) + return outs + +# Provided by Alexander Veysov + + +def int2float(sound): + abs_max = np.abs(sound).max() + sound = sound.astype('float32') + if abs_max > 0: + sound *= 1/32768 + sound = sound.squeeze() # depends on the use case + return sound + + +FORMAT = pyaudio.paInt16 +CHANNELS = 1 +SAMPLE_RATE = 16000 +CHUNK = int(SAMPLE_RATE / 10) + +audio = pyaudio.PyAudio() + + +class VADState(Enum): + QUIET = 1 + STARTING = 2 + SPEAKING = 3 + STOPPING = 4 + class BaseTransportService(): @@ -31,7 +85,23 @@ class BaseTransportService(): self._speaker_enabled = kwargs.get("speaker_enabled") or False self._speaker_sample_rate = kwargs.get("speaker_sample_rate") or 16000 self._fps = kwargs.get("fps") or 8 - + self._vad_start_s = kwargs.get("vad_start_s") or 0.2 + self._vad_stop_s = kwargs.get("vad_stop_s") or 0.8 + self._context = kwargs.get("context") or [] + self._vad_enabled = kwargs.get("vad_enabled") or False + + if self._vad_enabled and self._speaker_enabled: + raise Exception("Sorry, you can't use speaker_enabled and vad_enabled at the same time. Please set one to False.") + + self._vad_samples = 1536 + vad_frame_s = self._vad_samples / SAMPLE_RATE + self._vad_start_frames = round(self._vad_start_s / vad_frame_s) + self._vad_stop_frames = round(self._vad_stop_s / vad_frame_s) + self._vad_starting_count = 0 + self._vad_stopping_count = 0 + self._vad_state = VADState.QUIET + self._user_is_speaking = False + duration_minutes = kwargs.get("duration_minutes") or 10 self._expiration = time.time() + duration_minutes * 60 @@ -66,6 +136,10 @@ class BaseTransportService(): if self._speaker_enabled: self._receive_audio_thread = threading.Thread(target=self._receive_audio, daemon=True) self._receive_audio_thread.start() + + if self._vad_enabled: + self._vad_thread = threading.Thread(target=self._vad, daemon=True) + self._vad_thread.start() try: while ( @@ -89,6 +163,10 @@ class BaseTransportService(): if self._speaker_enabled: self._receive_audio_thread.join() + + if self._vad_enabled: + self._vad_thread.join() + def _post_run(self): # Note that this function must be idempotent! It can be called multiple times @@ -121,7 +199,57 @@ class BaseTransportService(): @abstractmethod def _prerun(self): pass - + + def _vad(self): + # CB: Starting silero VAD stuff + # TODO-CB: Probably need to force virtual speaker creation if we're + # going to build this in? + # TODO-CB: pyaudio installation + while not self._stop_threads.is_set(): + audio_chunk = self.read_audio_frames(self._vad_samples) + audio_int16 = np.frombuffer(audio_chunk, np.int16) + audio_float32 = int2float(audio_int16) + new_confidence = model( + torch.from_numpy(audio_float32), 16000).item() + speaking = new_confidence > 0.5 + + if speaking: + match self._vad_state: + case VADState.QUIET: + self._vad_state = VADState.STARTING + self._vad_starting_count = 1 + case VADState.STARTING: + self._vad_starting_count += 1 + case VADState.STOPPING: + self._vad_state = VADState.SPEAKING + self._vad_stopping_count = 0 + else: + match self._vad_state: + case VADState.STARTING: + self._vad_state = VADState.QUIET + self._vad_starting_count = 0 + case VADState.SPEAKING: + self._vad_state = VADState.STOPPING + self._vad_stopping_count = 1 + case VADState.STOPPING: + self._vad_stopping_count += 1 + + if self._vad_state == VADState.STARTING and self._vad_starting_count >= self._vad_start_frames: + asyncio.run_coroutine_threadsafe( + self.receive_queue.put( + UserStartedSpeakingFrame()), self._loop + ) + # self.interrupt() + self._vad_state = VADState.SPEAKING + self._vad_starting_count = 0 + if self._vad_state == VADState.STOPPING and self._vad_stopping_count >= self._vad_stop_frames: + asyncio.run_coroutine_threadsafe( + self.receive_queue.put( + UserStoppedSpeakingFrame()), self._loop + ) + self._vad_state = VADState.QUIET + self._vad_stopping_count = 0 + async def _marshal_frames(self): while True: frame: QueueFrame | list = await self.send_queue.get() diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index c4266406d..2b8416336 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -31,6 +31,7 @@ class DailyTransportService(BaseTransportService, EventHandler): _speaker_enabled: bool _speaker_sample_rate: int + _vad_enabled: bool # This is necessary to override EventHandler's __new__ method. def __new__(cls, *args, **kwargs): @@ -142,7 +143,7 @@ class DailyTransportService(BaseTransportService, EventHandler): "camera", width=self._camera_width, height=self._camera_height, color_format="RGB" ) - if self._speaker_enabled: + if self._speaker_enabled or self._vad_enabled: self._speaker: VirtualSpeakerDevice = Daily.create_speaker_device( "speaker", sample_rate=self._speaker_sample_rate, channels=1 ) diff --git a/src/examples/foundational/06-listen-and-respond.py b/src/examples/foundational/06-listen-and-respond.py index fa5e077cc..7cceb607d 100644 --- a/src/examples/foundational/06-listen-and-respond.py +++ b/src/examples/foundational/06-listen-and-respond.py @@ -3,8 +3,9 @@ import os from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.azure_ai_services import AzureLLMService, AzureTTSService +from dailyai.services.ai_services import FrameLogger from dailyai.queue_aggregators import LLMAssistantContextAggregator, LLMContextAggregator, LLMUserContextAggregator -from examples.foundational.support.runner import configure +from support.runner import configure async def main(room_url: str, token): @@ -16,7 +17,8 @@ async def main(room_url: str, token): start_transcription=True, mic_enabled=True, mic_sample_rate=16000, - camera_enabled=False + camera_enabled=False, + vad_enabled=True ) llm = AzureLLMService( @@ -26,7 +28,8 @@ async def main(room_url: str, token): tts = AzureTTSService( api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - + fl = FrameLogger("Inner") + fl2 = FrameLogger("Outer") @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): await tts.say("Hi, I'm listening!", transport.send_queue) @@ -44,14 +47,20 @@ async def main(room_url: str, token): await tts.run_to_queue( transport.send_queue, tma_out.run( - llm.run( + fl2.run( + llm.run( tma_in.run( - transport.get_receive_frames() + fl.run( + transport.get_receive_frames() + ) + ) ) ) - ) - ) + ) + ) + + transport.transcription_settings["extra"]["endpointing"] = True transport.transcription_settings["extra"]["punctuate"] = True await asyncio.gather(transport.run(), handle_transcriptions())