diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..cd6220df2 --- /dev/null +++ b/LICENSE @@ -0,0 +1,24 @@ +BSD 2-Clause License + +Copyright (c) 2024, Daily + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index c1122778b..bc585fa3a 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Build conversational, multi-modal AI apps with real-time voice and video, like this: -_Demo Video_ +_Demo Video to come_ With built-in support for many of the best AI platforms (or [add your own](/docs)): @@ -15,22 +15,6 @@ With built-in support for many of the best AI platforms (or [add your own](/docs ## Step 1: Get Started -Installation here. Also sign up for a Daily account, I guess? also we need an ENV - -Requires python 3.11 or later. Don't forget virtualenv - -pip install vs download and build? - -## Step 2: Build Things - -Once you've got the SDK working, head over to the [docs folder](/docs) to start building! - ---- - -# Old Readme - -This SDK can help you build applications that participate in WebRTC meetings and use various AI services to interact with other participants. - ## Build/Install _Note that you may need to set up a virtual environment before following the instructions below. For instance, you might need to run the following from the root of the repo:_ @@ -66,23 +50,6 @@ Tou can run the simple sample like so: ``` python src/examples/theoretical-to-real/01-say-one-thing.py -u -k ``` - -Note that the sample uses Azure's TTS and LLM services. You'll need to set the following environment variables for the sample to work: - -``` -AZURE_SPEECH_SERVICE_KEY -AZURE_SPEECH_SERVICE_REGION -AZURE_CHATGPT_KEY -AZURE_CHATGPT_ENDPOINT -AZURE_CHATGPT_DEPLOYMENT_ID -``` - -If you have those environment variables stored in an .env file, you can quickly load them into your terminal's environment by running this: - -```bash -export $(grep -v '^#' .env | xargs) -``` - ## Overview The Daily AI SDK allows you to build applications that can participate in WebRTC sessions and interact with AI Services. Some examples of what you can build with this: diff --git a/pyproject.toml b/pyproject.toml index 7603f8500..e6aa7e5a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,10 +13,13 @@ dependencies = [ "fal", "faster_whisper", "google-cloud-texttospeech", + "numpy", "openai", "Pillow", "pyht", "python-dotenv", + "torch", + "pyaudio", "typing-extensions" ] diff --git a/src/dailyai/queue_aggregators.py b/src/dailyai/queue_aggregators.py index 3d9d93c39..5460bbab5 100644 --- a/src/dailyai/queue_aggregators.py +++ b/src/dailyai/queue_aggregators.py @@ -61,7 +61,8 @@ class LLMContextAggregator(AIService): # TODO: split up transcription by participant if self.complete_sentences: - # type: ignore -- the linter thinks this isn't a TextQueueFrame, even though we check it above + # type: ignore -- the linter thinks this isn't a TextQueueFrame, even + # though we check it above self.sentence += frame.text if self.sentence.endswith((".", "?", "!")): self.messages.append( @@ -71,10 +72,9 @@ class LLMContextAggregator(AIService): print(f"{message['role']}: {message['content']}") yield LLMMessagesQueueFrame(self.messages) else: - # type: ignore -- the linter thinks this isn't a TextQueueFrame, even though we check it above + # type: ignore -- the linter thinks this isn't a TextQueueFrame, even + # though we check it above self.messages.append({"role": self.role, "content": frame.text}) - for message in self.messages: - print(f"{message['role']}: {message['content']}") yield LLMMessagesQueueFrame(self.messages) async def finalize(self) -> AsyncGenerator[QueueFrame, None]: @@ -91,8 +91,7 @@ class LLMUserContextAggregator(LLMContextAggregator): messages: list[dict], bot_participant_id=None, complete_sentences=True): - super().__init__(messages, "user", bot_participant_id, - complete_sentences, pass_through=False) + super().__init__(messages, "user", bot_participant_id, complete_sentences, pass_through=False) class LLMAssistantContextAggregator(LLMContextAggregator): diff --git a/src/dailyai/queue_frame.py b/src/dailyai/queue_frame.py index 48be056c7..afd0ac596 100644 --- a/src/dailyai/queue_frame.py +++ b/src/dailyai/queue_frame.py @@ -69,3 +69,9 @@ class LLMMessagesQueueFrame(QueueFrame): class AppMessageQueueFrame(QueueFrame): message: Any participantId: str + +class UserStartedSpeakingFrame(QueueFrame): + pass + +class UserStoppedSpeakingFrame(QueueFrame): + pass \ No newline at end of file diff --git a/src/dailyai/services/base_transport_service.py b/src/dailyai/services/base_transport_service.py index a13c63498..22a6d5777 100644 --- a/src/dailyai/services/base_transport_service.py +++ b/src/dailyai/services/base_transport_service.py @@ -2,10 +2,15 @@ from abc import abstractmethod import asyncio import itertools import logging +import numpy as np +import pyaudio +import torch +import torchaudio import queue import threading import time from typing import AsyncGenerator +from enum import Enum from dailyai.queue_frame import ( AudioQueueFrame, @@ -15,9 +20,58 @@ from dailyai.queue_frame import ( QueueFrame, SpriteQueueFrame, StartStreamQueueFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame ) +torch.set_num_threads(1) + +model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad', + model='silero_vad', + force_reload=False) + +(get_speech_timestamps, + save_audio, + read_audio, + VADIterator, + collect_chunks) = utils + +# Taken from utils_vad.py + + +def validate(model, + inputs: torch.Tensor): + with torch.no_grad(): + outs = model(inputs) + return outs + +# Provided by Alexander Veysov + + +def int2float(sound): + abs_max = np.abs(sound).max() + sound = sound.astype('float32') + if abs_max > 0: + sound *= 1/32768 + sound = sound.squeeze() # depends on the use case + return sound + + +FORMAT = pyaudio.paInt16 +CHANNELS = 1 +SAMPLE_RATE = 16000 +CHUNK = int(SAMPLE_RATE / 10) + +audio = pyaudio.PyAudio() + + +class VADState(Enum): + QUIET = 1 + STARTING = 2 + SPEAKING = 3 + STOPPING = 4 + class BaseTransportService(): def __init__( @@ -32,7 +86,23 @@ class BaseTransportService(): self._speaker_enabled = kwargs.get("speaker_enabled") or False self._speaker_sample_rate = kwargs.get("speaker_sample_rate") or 16000 self._fps = kwargs.get("fps") or 8 - + self._vad_start_s = kwargs.get("vad_start_s") or 0.2 + self._vad_stop_s = kwargs.get("vad_stop_s") or 0.8 + self._context = kwargs.get("context") or [] + self._vad_enabled = kwargs.get("vad_enabled") or False + + if self._vad_enabled and self._speaker_enabled: + raise Exception("Sorry, you can't use speaker_enabled and vad_enabled at the same time. Please set one to False.") + + self._vad_samples = 1536 + vad_frame_s = self._vad_samples / SAMPLE_RATE + self._vad_start_frames = round(self._vad_start_s / vad_frame_s) + self._vad_stop_frames = round(self._vad_stop_s / vad_frame_s) + self._vad_starting_count = 0 + self._vad_stopping_count = 0 + self._vad_state = VADState.QUIET + self._user_is_speaking = False + duration_minutes = kwargs.get("duration_minutes") or 10 self._expiration = time.time() + duration_minutes * 60 @@ -71,6 +141,10 @@ class BaseTransportService(): self._receive_audio_thread = threading.Thread( target=self._receive_audio, daemon=True) self._receive_audio_thread.start() + + if self._vad_enabled: + self._vad_thread = threading.Thread(target=self._vad, daemon=True) + self._vad_thread.start() try: while ( @@ -81,6 +155,10 @@ class BaseTransportService(): except Exception as e: self._logger.error(f"Exception {e}") raise e + finally: + # Do anything that must be done to clean up + self._post_run() + self._stop_threads.set() await self.send_queue.put(EndStreamQueueFrame()) @@ -90,6 +168,15 @@ class BaseTransportService(): if self._speaker_enabled: self._receive_audio_thread.join() + + if self._vad_enabled: + self._vad_thread.join() + + + def _post_run(self): + # Note that this function must be idempotent! It can be called multiple times + # if, for example, a keyboard interrupt occurs. + pass def stop(self): self._stop_threads.set() @@ -117,7 +204,57 @@ class BaseTransportService(): @abstractmethod def _prerun(self): pass - + + def _vad(self): + # CB: Starting silero VAD stuff + # TODO-CB: Probably need to force virtual speaker creation if we're + # going to build this in? + # TODO-CB: pyaudio installation + while not self._stop_threads.is_set(): + audio_chunk = self.read_audio_frames(self._vad_samples) + audio_int16 = np.frombuffer(audio_chunk, np.int16) + audio_float32 = int2float(audio_int16) + new_confidence = model( + torch.from_numpy(audio_float32), 16000).item() + speaking = new_confidence > 0.5 + + if speaking: + match self._vad_state: + case VADState.QUIET: + self._vad_state = VADState.STARTING + self._vad_starting_count = 1 + case VADState.STARTING: + self._vad_starting_count += 1 + case VADState.STOPPING: + self._vad_state = VADState.SPEAKING + self._vad_stopping_count = 0 + else: + match self._vad_state: + case VADState.STARTING: + self._vad_state = VADState.QUIET + self._vad_starting_count = 0 + case VADState.SPEAKING: + self._vad_state = VADState.STOPPING + self._vad_stopping_count = 1 + case VADState.STOPPING: + self._vad_stopping_count += 1 + + if self._vad_state == VADState.STARTING and self._vad_starting_count >= self._vad_start_frames: + asyncio.run_coroutine_threadsafe( + self.receive_queue.put( + UserStartedSpeakingFrame()), self._loop + ) + # self.interrupt() + self._vad_state = VADState.SPEAKING + self._vad_starting_count = 0 + if self._vad_state == VADState.STOPPING and self._vad_stopping_count >= self._vad_stop_frames: + asyncio.run_coroutine_threadsafe( + self.receive_queue.put( + UserStoppedSpeakingFrame()), self._loop + ) + self._vad_state = VADState.QUIET + self._vad_stopping_count = 0 + async def _marshal_frames(self): while True: frame: QueueFrame | list = await self.send_queue.get() diff --git a/src/dailyai/services/daily_transport_service.py b/src/dailyai/services/daily_transport_service.py index fed6fd661..5b059dca0 100644 --- a/src/dailyai/services/daily_transport_service.py +++ b/src/dailyai/services/daily_transport_service.py @@ -1,8 +1,8 @@ import asyncio import inspect import logging +import signal import threading -import time import types from functools import partial @@ -11,7 +11,7 @@ from dailyai.queue_frame import ( TranscriptionQueueFrame, ) -from threading import Thread, Event +from threading import Event from daily import ( EventHandler, @@ -31,6 +31,7 @@ class DailyTransportService(BaseTransportService, EventHandler): _speaker_enabled: bool _speaker_sample_rate: int + _vad_enabled: bool # This is necessary to override EventHandler's __new__ method. def __new__(cls, *args, **kwargs): @@ -45,8 +46,7 @@ class DailyTransportService(BaseTransportService, EventHandler): start_transcription: bool = False, **kwargs, ): - # This will call BaseTransportService.__init__ method, not EventHandler - super().__init__(**kwargs) + super().__init__(**kwargs) # This will call BaseTransportService.__init__ method, not EventHandler self._room_url: str = room_url self._bot_name: str = bot_name @@ -146,59 +146,76 @@ class DailyTransportService(BaseTransportService, EventHandler): "camera", width=self._camera_width, height=self._camera_height, color_format="RGB" ) - if self._speaker_enabled: + if self._speaker_enabled or self._vad_enabled: self._speaker: VirtualSpeakerDevice = Daily.create_speaker_device( "speaker", sample_rate=self._speaker_sample_rate, channels=1 ) Daily.select_speaker_device("speaker") self.client.set_user_name(self._bot_name) - self.client.join(self._room_url, self._token, - completion=self.call_joined) + + self.client.join( + self._room_url, + self._token, + completion=self.call_joined, + client_settings={ + "inputs": { + "camera": { + "isEnabled": True, + "settings": { + "deviceId": "camera", + }, + }, + "microphone": { + "isEnabled": True, + "settings": { + "deviceId": "mic", + "customConstraints": { + "autoGainControl": {"exact": False}, + "echoCancellation": {"exact": False}, + "noiseSuppression": {"exact": False}, + }, + }, + }, + }, + "publishing": { + "camera": { + "sendSettings": { + "maxQuality": "low", + "encodings": { + "low": { + "maxBitrate": 250000, + "scaleResolutionDownBy": 1.333, + "maxFramerate": 8, + } + }, + } + } + }, + }, + ) self._my_participant_id = self.client.participants()["local"]["id"] - self.client.update_inputs( - { - "camera": { - "isEnabled": True, - "settings": { - "deviceId": "camera", - }, - }, - "microphone": { - "isEnabled": True, - "settings": { - "deviceId": "mic", - "customConstraints": { - "autoGainControl": {"exact": False}, - "echoCancellation": {"exact": False}, - "noiseSuppression": {"exact": False}, - }, - }, - }, + self.client.update_subscription_profiles({ + "base": { + "camera": "unsubscribed", } - ) - - self.client.update_publishing( - { - "camera": { - "sendSettings": { - "maxQuality": "low", - "encodings": { - "low": { - "maxBitrate": 250000, - "scaleResolutionDownBy": 1.333, - "maxFramerate": 8, - } - }, - } - } - } - ) + }) if self._token and self._start_transcription: self.client.start_transcription(self.transcription_settings) + self.original_sigint_handler = signal.getsignal(signal.SIGINT) + signal.signal(signal.SIGINT, self.process_interrupt_handler) + + def process_interrupt_handler(self, signum, frame): + self._post_run() + if callable(self.original_sigint_handler): + self.original_sigint_handler(signum, frame) + + def _post_run(self): + self.client.leave() + def on_first_other_participant_joined(self): pass diff --git a/src/dailyai/services/deepgram_ai_service.py b/src/dailyai/services/deepgram_ai_service.py new file mode 100644 index 000000000..1ede1c35d --- /dev/null +++ b/src/dailyai/services/deepgram_ai_service.py @@ -0,0 +1,36 @@ +import os +import aiohttp +import requests + +from dailyai.services.ai_services import TTSService + + +class DeepgramAIService(TTSService): + def __init__( + self, + *, + aiohttp_session: aiohttp.ClientSession, + api_key, + voice, + sample_rate=16000 + ): + super().__init__() + + self._api_key = api_key + self._voice = voice + self._sample_rate = sample_rate + self._aiohttp_session = aiohttp_session + + async def run_tts(self, sentence): + self.logger.info(f"Running deepgram tts for {sentence}") + base_url = "https://api.beta.deepgram.com/v1/speak" + request_url = f"{base_url}?model={self._voice}&encoding=linear16&container=none&sample_rate={self._sample_rate}" + headers = {"authorization": f"token {self._api_key}", "Content-Type": "application/json"} + data = {"text": sentence} + + async with self._aiohttp_session.post( + request_url, headers=headers, json=data + ) as r: + async for chunk in r.content: + if chunk: + yield chunk diff --git a/src/dailyai/services/fal_ai_services.py b/src/dailyai/services/fal_ai_services.py index ca1f93d89..9464b46dd 100644 --- a/src/dailyai/services/fal_ai_services.py +++ b/src/dailyai/services/fal_ai_services.py @@ -13,7 +13,13 @@ from dailyai.services.ai_services import ImageGenService class FalImageGenService(ImageGenService): - def __init__(self, *, image_size, aiohttp_session: aiohttp.ClientSession, key_id=None, key_secret=None): + def __init__( + self, + *, + image_size, + aiohttp_session: aiohttp.ClientSession, + key_id=None, + key_secret=None): super().__init__(image_size) self._aiohttp_session = aiohttp_session if key_id: diff --git a/src/dailyai/services/local_transport_service.py b/src/dailyai/services/local_transport_service.py index 91af6763c..7538d3b06 100644 --- a/src/dailyai/services/local_transport_service.py +++ b/src/dailyai/services/local_transport_service.py @@ -22,11 +22,15 @@ class LocalTransportService(BaseTransportService): async def _write_frame_to_tkinter(self, frame: bytes): data = f"P6 {self._camera_width} {self._camera_height} 255 ".encode() + frame - photo = tk.PhotoImage(width=self._camera_width, height=self._camera_height, data=data, format="PPM") + photo = tk.PhotoImage( + width=self._camera_width, + height=self._camera_height, + data=data, + format="PPM") self._image_label.config(image=photo) # This holds a reference to the photo, preventing it from being garbage collected. - self._image_label.image = photo # type: ignore + self._image_label.image = photo # type: ignore def write_frame_to_camera(self, frame: bytes): if self._camera_enabled and self._loop: diff --git a/src/dailyai/services/ollama_ai_services.py b/src/dailyai/services/ollama_ai_services.py new file mode 100644 index 000000000..c065e0b7b --- /dev/null +++ b/src/dailyai/services/ollama_ai_services.py @@ -0,0 +1,42 @@ +from openai import AsyncOpenAI + +import json +from collections.abc import AsyncGenerator + +from dailyai.services.ai_services import LLMService + + +class OLLamaLLMService(LLMService): + def __init__(self, model="llama2", base_url='http://localhost:11434/v1'): + super().__init__() + self._model = model + self._client = AsyncOpenAI(api_key="ollama", base_url=base_url) + + async def get_response(self, messages, stream): + return await self._client.chat.completions.create( + stream=stream, + messages=messages, + model=self._model + ) + + async def run_llm_async(self, messages) -> AsyncGenerator[str, None]: + messages_for_log = json.dumps(messages) + self.logger.debug(f"Generating chat via openai: {messages_for_log}") + + chunks = await self._client.chat.completions.create(model=self._model, stream=True, messages=messages) + async for chunk in chunks: + if len(chunk.choices) == 0: + continue + + if chunk.choices[0].delta.content: + yield chunk.choices[0].delta.content + + async def run_llm(self, messages) -> str | None: + messages_for_log = json.dumps(messages) + self.logger.debug(f"Generating chat via openai: {messages_for_log}") + + response = await self._client.chat.completions.create(model=self._model, stream=False, messages=messages) + if response and len(response.choices) > 0: + return response.choices[0].message.content + else: + return None diff --git a/src/dailyai/services/open_ai_services.py b/src/dailyai/services/open_ai_services.py index f929985c8..1b18b8211 100644 --- a/src/dailyai/services/open_ai_services.py +++ b/src/dailyai/services/open_ai_services.py @@ -1,15 +1,12 @@ -import requests import aiohttp -import asyncio from PIL import Image import io from openai import AsyncOpenAI -import os import json from collections.abc import AsyncGenerator -from dailyai.services.ai_services import AIService, TTSService, LLMService, ImageGenService +from dailyai.services.ai_services import LLMService, ImageGenService class OpenAILLMService(LLMService): diff --git a/src/dailyai/services/to_be_updated/playht_ai_service.py b/src/dailyai/services/playht_ai_service.py similarity index 75% rename from src/dailyai/services/to_be_updated/playht_ai_service.py rename to src/dailyai/services/playht_ai_service.py index 4ba9ddc86..350e5cb88 100644 --- a/src/dailyai/services/to_be_updated/playht_ai_service.py +++ b/src/dailyai/services/playht_ai_service.py @@ -1,36 +1,40 @@ import io -import os import struct from pyht import Client -from dotenv import load_dotenv from pyht.client import TTSOptions from pyht.protos.api_pb2 import Format -from services.ai_service import AIService +from dailyai.services.ai_services import TTSService -class PlayHTAIService(AIService): - def __init__(self, **kwargs): - super().__init__(**kwargs) +class PlayHTAIService(TTSService): - self.speech_key = os.getenv("PLAY_HT_KEY") or '' - self.user_id = os.getenv("PLAY_HT_USER_ID") or '' + def __init__( + self, + *, + api_key, + user_id, + voice_url + ): + super().__init__() + + self.speech_key = api_key + self.user_id = user_id self.client = Client( user_id=self.user_id, api_key=self.speech_key, ) self.options = TTSOptions( - voice="s3://voice-cloning-zero-shot/820da3d2-3a3b-42e7-844d-e68db835a206/sarah/manifest.json", + voice=voice_url, sample_rate=16000, quality="higher", format=Format.FORMAT_WAV) - def close(self): - super().close() + def __del__(self): self.client.close() - def run_tts(self, sentence): + async def run_tts(self, sentence): b = bytearray() in_header = True for chunk in self.client.tts(sentence, self.options): diff --git a/src/dailyai/services/to_be_updated/deepgram_ai_service.py b/src/dailyai/services/to_be_updated/deepgram_ai_service.py deleted file mode 100644 index b4569e6cd..000000000 --- a/src/dailyai/services/to_be_updated/deepgram_ai_service.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import requests - -from services.ai_service import AIService -from PIL import Image - - -class DeepgramAIService(AIService): - def __init__(self, **kwargs): - super().__init__(**kwargs) - - self.api_key = os.getenv("DEEPGRAM_API_KEY") - - def get_mic_sample_rate(self): - return 24000 - - def run_tts(self, sentence): - self.logger.info(f"Running deepgram tts for {sentence}") - base_url = "https://api.beta.deepgram.com/v1/speak" - # move this to an environment variable - voice = os.getenv("DEEPGRAM_VOICE") or "alpha-apollo-en-v1" - request_url = f"{base_url}?model={voice}&encoding=linear16&container=none" - headers = {"authorization": f"token {self.api_key}"} - - r = requests.post(request_url, headers=headers, data=sentence) - self.logger.info( - f"audio fetch status code: {r.status_code}, content length: {len(r.content)}" - ) - yield r.content diff --git a/src/examples/foundational/01-say-one-thing.py b/src/examples/foundational/01-say-one-thing.py index 989da84af..9df54d708 100644 --- a/src/examples/foundational/01-say-one-thing.py +++ b/src/examples/foundational/01-say-one-thing.py @@ -4,9 +4,11 @@ import os from dailyai.services.daily_transport_service import DailyTransportService from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService +from dailyai.services.playht_ai_service import PlayHTAIService from examples.foundational.support.runner import configure + async def main(room_url): async with aiohttp.ClientSession() as session: # create a transport service object using environment variables for @@ -25,11 +27,23 @@ async def main(room_url): meeting_duration_minutes, mic_enabled=True ) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + + """ + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + """ + tts = PlayHTAIService( + api_key=os.getenv("PLAY_HT_API_KEY"), + user_id=os.getenv("PLAY_HT_USER_ID"), + voice_url=os.getenv("PLAY_HT_VOICE_URL"), + ) # Register an event handler so we can play the audio when the participant joins. @transport.event_handler("on_participant_joined") async def on_participant_joined(transport, participant): + nonlocal tts if participant["info"]["isLocal"]: return @@ -42,6 +56,7 @@ async def main(room_url): await transport.stop_when_done() await transport.run() + del(tts) if __name__ == "__main__": diff --git a/src/examples/foundational/02-llm-say-one-thing.py b/src/examples/foundational/02-llm-say-one-thing.py index 7a30a906f..b15023380 100644 --- a/src/examples/foundational/02-llm-say-one-thing.py +++ b/src/examples/foundational/02-llm-say-one-thing.py @@ -11,6 +11,7 @@ from dailyai.services.deepgram_ai_services import DeepgramTTSService from dailyai.services.open_ai_services import OpenAILLMService from examples.foundational.support.runner import configure + async def main(room_url): async with aiohttp.ClientSession() as session: meeting_duration_minutes = 1 @@ -22,12 +23,18 @@ async def main(room_url): mic_enabled=True ) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID")) # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) # tts = DeepgramTTSService(aiohttp_session=session, api_key=os.getenv("DEEPGRAM_API_KEY"), voice=os.getenv("DEEPGRAM_VOICE")) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - #llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + # llm = OpenAILLMService(api_key=os.getenv("OPENAI_CHATGPT_API_KEY")) messages = [{ "role": "system", "content": "You are an LLM in a WebRTC session, and this is a 'hello world' demo. Say hello to the world." diff --git a/src/examples/foundational/03-still-frame.py b/src/examples/foundational/03-still-frame.py index 06d40448b..de368d5d6 100644 --- a/src/examples/foundational/03-still-frame.py +++ b/src/examples/foundational/03-still-frame.py @@ -28,7 +28,11 @@ async def main(room_url): camera_height=1024 ) - imagegen = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + imagegen = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) # imagegen = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024") # imagegen = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL")) diff --git a/src/examples/foundational/04-utterance-and-speech.py b/src/examples/foundational/04-utterance-and-speech.py index 18e7c2ff7..17bd32797 100644 --- a/src/examples/foundational/04-utterance-and-speech.py +++ b/src/examples/foundational/04-utterance-and-speech.py @@ -10,6 +10,7 @@ from dailyai.services.elevenlabs_ai_service import ElevenLabsTTSService from examples.foundational.support.runner import configure + async def main(room_url: str): async with aiohttp.ClientSession() as session: transport = DailyTransportService( @@ -22,9 +23,17 @@ async def main(room_url: str): camera_enabled=False ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - azure_tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - elevenlabs_tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id=os.getenv("ELEVENLABS_VOICE_ID")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + azure_tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) + elevenlabs_tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id=os.getenv("ELEVENLABS_VOICE_ID")) messages = [{"role": "system", "content": "tell the user a joke about llamas"}] diff --git a/src/examples/foundational/05-sync-speech-and-image.py b/src/examples/foundational/05-sync-speech-and-image.py index 55e4ec9f7..dfef2bc15 100644 --- a/src/examples/foundational/05-sync-speech-and-image.py +++ b/src/examples/foundational/05-sync-speech-and-image.py @@ -11,6 +11,7 @@ from dailyai.services.open_ai_services import OpenAIImageGenService from examples.foundational.support.runner import configure + async def main(room_url): async with aiohttp.ClientSession() as session: meeting_duration_minutes = 5 @@ -26,11 +27,21 @@ async def main(room_url): camera_height=1024 ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV") + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="ErXwobaYiN019PkySvjV") # tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + dalle = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) # dalle = OpenAIImageGenService(aiohttp_session=session, api_key=os.getenv("OPENAI_DALLE_API_KEY"), image_size="1024x1024") # dalle = AzureImageGenServiceREST(image_size="1024x1024", aiohttp_session=session, api_key=os.getenv("AZURE_DALLE_API_KEY"), endpoint=os.getenv("AZURE_DALLE_ENDPOINT"), model=os.getenv("AZURE_DALLE_MODEL")) diff --git a/src/examples/foundational/06-listen-and-respond.py b/src/examples/foundational/06-listen-and-respond.py index d02ff0aab..12deb383b 100644 --- a/src/examples/foundational/06-listen-and-respond.py +++ b/src/examples/foundational/06-listen-and-respond.py @@ -88,11 +88,12 @@ Once you have collected all of that information, respond with a JSON object cont ) ) ) - + transport.transcription_settings["extra"]["endpointing"] = True transport.transcription_settings["extra"]["punctuate"] = True await asyncio.gather(transport.run(), handle_transcriptions()) + if __name__ == "__main__": (url, token) = configure() asyncio.run(main(url, token)) diff --git a/src/examples/foundational/06a-image-sync.py b/src/examples/foundational/06a-image-sync.py index fbef24364..032d828c1 100644 --- a/src/examples/foundational/06a-image-sync.py +++ b/src/examples/foundational/06a-image-sync.py @@ -18,6 +18,7 @@ from dailyai.services.fal_ai_services import FalImageGenService from examples.foundational.support.runner import configure + class ImageSyncAggregator(AIService): def __init__(self, speaking_path: str, waiting_path: str): self._speaking_image = Image.open(speaking_path) @@ -46,9 +47,18 @@ async def main(room_url: str, token): transport._mic_enabled = True transport._mic_sample_rate = 16000 - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - img = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) + img = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) async def get_images(): get_speaking_task = asyncio.create_task( diff --git a/src/examples/foundational/07-interruptible.py b/src/examples/foundational/07-interruptible.py index 53f0eb326..5421b480f 100644 --- a/src/examples/foundational/07-interruptible.py +++ b/src/examples/foundational/07-interruptible.py @@ -25,10 +25,13 @@ async def main(room_url: str, token): camera_enabled=False, ) - llm = OpenAILLMService(api_key=os.getenv( - "OPENAI_CHATGPT_API_KEY"), model="gpt-4", tools=None) - tts = AzureTTSService(api_key=os.getenv( - "AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) async def run_response(user_speech, tma_in, tma_out): await tts.run_to_queue( diff --git a/src/examples/foundational/08-bots-arguing.py b/src/examples/foundational/08-bots-arguing.py index 95fc62e40..a5329086e 100644 --- a/src/examples/foundational/08-bots-arguing.py +++ b/src/examples/foundational/08-bots-arguing.py @@ -10,7 +10,8 @@ from dailyai.queue_frame import AudioQueueFrame, ImageQueueFrame from examples.foundational.support.runner import configure -async def main(room_url:str): + +async def main(room_url: str): async with aiohttp.ClientSession() as session: transport = DailyTransportService( room_url, @@ -24,16 +25,30 @@ async def main(room_url:str): camera_height=1024 ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts1 = AzureTTSService(api_key=os.getenv("AZURE_SPEECH_API_KEY"), region=os.getenv("AZURE_SPEECH_REGION")) - tts2 = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl") - dalle = FalImageGenService(image_size="1024x1024", aiohttp_session=session, key_id=os.getenv("FAL_KEY_ID"), key_secret=os.getenv("FAL_KEY_SECRET")) + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts1 = AzureTTSService( + api_key=os.getenv("AZURE_SPEECH_API_KEY"), + region=os.getenv("AZURE_SPEECH_REGION")) + tts2 = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="jBpfuIE2acCO8z3wKNLl") + dalle = FalImageGenService( + image_size="1024x1024", + aiohttp_session=session, + key_id=os.getenv("FAL_KEY_ID"), + key_secret=os.getenv("FAL_KEY_SECRET")) bot1_messages = [ {"role": "system", "content": "You are a stern librarian. You strongly believe that a hot dog is a sandwich. Start by stating this fact in a few sentences, then be prepared to debate this with the user. You shouldn't ever compromise on the fundamental truth that a hot dog is a sandwich. Your responses should only be a few sentences long."}, ] bot2_messages = [ - {"role": "system", "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, + { + "role": "system", + "content": "You are a silly cat, and you strongly believe that a hot dog is not a sandwich. Debate this with the user, only responding with a few sentences. Don't ever accept that a hot dog is a sandwich."}, ] async def get_bot1_statement(): diff --git a/src/examples/foundational/10-wake-word.py b/src/examples/foundational/10-wake-word.py index ccf706d50..54db2468c 100644 --- a/src/examples/foundational/10-wake-word.py +++ b/src/examples/foundational/10-wake-word.py @@ -71,7 +71,7 @@ class TranscriptFilter(AIService): class NameCheckFilter(AIService): - def __init__(self, names:list[str]): + def __init__(self, names: list[str]): self.names = names self.sentence = "" @@ -123,8 +123,14 @@ async def main(room_url: str, token): transport._camera_width = 720 transport._camera_height = 1280 - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="jBpfuIE2acCO8z3wKNLl") + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="jBpfuIE2acCO8z3wKNLl") isa = ImageSyncAggregator() @transport.event_handler("on_first_other_participant_joined") diff --git a/src/examples/foundational/11-sound-effects.py b/src/examples/foundational/11-sound-effects.py index 743120f23..954f363d7 100644 --- a/src/examples/foundational/11-sound-effects.py +++ b/src/examples/foundational/11-sound-effects.py @@ -14,7 +14,7 @@ from typing import AsyncGenerator from examples.foundational.support.runner import configure -logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever +logging.basicConfig(format=f"%(levelno)s %(asctime)s %(message)s") # or whatever logger = logging.getLogger("dailyai") logger.setLevel(logging.DEBUG) @@ -36,8 +36,6 @@ for file in sound_files: sounds[file] = audio_file.readframes(-1) - - class OutboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -50,6 +48,7 @@ class OutboundSoundEffectWrapper(AIService): else: yield frame + class InboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -75,14 +74,20 @@ async def main(room_url: str, token): camera_enabled=False ) - llm = AzureLLMService(api_key=os.getenv("AZURE_CHATGPT_API_KEY"), endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), model=os.getenv("AZURE_CHATGPT_MODEL")) - tts = ElevenLabsTTSService(aiohttp_session=session, api_key=os.getenv("ELEVENLABS_API_KEY"), voice_id="ErXwobaYiN019PkySvjV") - + llm = AzureLLMService( + api_key=os.getenv("AZURE_CHATGPT_API_KEY"), + endpoint=os.getenv("AZURE_CHATGPT_ENDPOINT"), + model=os.getenv("AZURE_CHATGPT_MODEL")) + tts = ElevenLabsTTSService( + aiohttp_session=session, + api_key=os.getenv("ELEVENLABS_API_KEY"), + voice_id="ErXwobaYiN019PkySvjV") @transport.event_handler("on_first_other_participant_joined") async def on_first_other_participant_joined(transport): await tts.say("Hi, I'm listening!", transport.send_queue) await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) + async def handle_transcriptions(): messages = [ {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, @@ -117,7 +122,6 @@ async def main(room_url: str, token): ) ) - transport.transcription_settings["extra"]["punctuate"] = True await asyncio.gather(transport.run(), handle_transcriptions()) diff --git a/src/examples/foundational/13-whisper-transcription.py b/src/examples/foundational/13-whisper-transcription.py index 147b20e22..726616c5f 100644 --- a/src/examples/foundational/13-whisper-transcription.py +++ b/src/examples/foundational/13-whisper-transcription.py @@ -5,6 +5,7 @@ from dailyai.services.whisper_ai_services import WhisperSTTService from examples.foundational.support.runner import configure + async def main(room_url: str): transport = DailyTransportService( room_url, diff --git a/src/examples/foundational/13a-whisper-local.py b/src/examples/foundational/13a-whisper-local.py index 0a30ad721..6c764c1a9 100644 --- a/src/examples/foundational/13a-whisper-local.py +++ b/src/examples/foundational/13a-whisper-local.py @@ -17,7 +17,7 @@ async def main(room_url: str): camera_enabled=False, speaker_enabled=True, duration_minutes=meeting_duration_minutes, - start_transcription = True + start_transcription=True ) stt = WhisperSTTService() transcription_output_queue = asyncio.Queue() diff --git a/src/examples/foundational/support/runner.py b/src/examples/foundational/support/runner.py index 94263c0a6..b4b7d4862 100644 --- a/src/examples/foundational/support/runner.py +++ b/src/examples/foundational/support/runner.py @@ -7,6 +7,7 @@ import requests from dotenv import load_dotenv load_dotenv() + def configure(): parser = argparse.ArgumentParser(description="Daily AI SDK Bot Sample") parser.add_argument( @@ -26,11 +27,11 @@ def configure(): key = args.apikey or os.getenv("DAILY_API_KEY") if not url: - raise Exception("No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.") - + raise Exception( + "No Daily room specified. use the -u/--url option from the command line, or set DAILY_SAMPLE_ROOM_URL in your environment to specify a Daily room URL.") + if not key: raise Exception("No Daily API key specified. use the -k/--apikey option from the command line, or set DAILY_API_KEY in your environment to specify a Daily API key, available from https://dashboard.daily.co/developers.") - # Create a meeting token for the given room with an expiration 1 hour in the future. room_name: str = urllib.parse.urlparse(url).path[1:] @@ -49,4 +50,4 @@ def configure(): token: str = res.json()["token"] - return (url, token) \ No newline at end of file + return (url, token) diff --git a/src/examples/internal/11a-dial-out.py b/src/examples/internal/11a-dial-out.py index 9d38299f4..b16c72ed1 100644 --- a/src/examples/internal/11a-dial-out.py +++ b/src/examples/internal/11a-dial-out.py @@ -30,8 +30,6 @@ for file in sound_files: sounds[file] = audio_file.readframes(-1) - - class OutboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -44,6 +42,7 @@ class OutboundSoundEffectWrapper(AIService): else: yield frame + class InboundSoundEffectWrapper(AIService): def __init__(self): pass @@ -81,6 +80,7 @@ async def main(room_url: str, token, phone): async def on_first_other_participant_joined(transport): await tts.say("Hi, I'm listening!", transport.send_queue) await transport.send_queue.put(AudioQueueFrame(sounds["ding1.wav"])) + async def handle_transcriptions(): messages = [ {"role": "system", "content": "You are a helpful LLM in a WebRTC call. Your goal is to demonstrate your capabilities in a succinct way. Your output will be converted to audio. Respond to what the user said in a creative and helpful way."}, @@ -124,7 +124,6 @@ async def main(room_url: str, token, phone): transport.start_recording() transport.dialout(phone) - transport.transcription_settings["extra"]["punctuate"] = True await asyncio.gather(transport.run(), handle_transcriptions())