From 2f59e38a7a240a3e0eb72cb1269f1897f3d0d362 Mon Sep 17 00:00:00 2001 From: chadbailey59 Date: Wed, 3 Apr 2024 10:48:11 -0500 Subject: [PATCH] Modularize tricky dependencies (#95) * removed pyaudio from threaded transport * modularized torch and torchaudio * modularized local transport * Working Dockerfile as well * docker updates for fly.io --- .dockerignore | 30 +++++++++++++ .gitignore | 1 + examples/server/Dockerfile => Dockerfile | 5 ++- examples/server/daily-bot-manager.py | 10 ++--- examples/starter-apps/translator.py | 3 +- pyproject.toml | 5 +-- src/dailyai/transports/local_transport.py | 8 +++- src/dailyai/transports/threaded_transport.py | 44 +++++++++----------- 8 files changed, 69 insertions(+), 37 deletions(-) create mode 100644 .dockerignore rename examples/server/Dockerfile => Dockerfile (82%) diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..01712f036 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,30 @@ +# flyctl launch added from .gitignore +**/.vscode +**/env +**/__pycache__ +**/*~ +**/venv +#*# + +# Distribution / packaging +**/.Python +**/build +**/develop-eggs +**/dist +**/downloads +**/eggs +**/.eggs +**/lib +**/lib64 +**/parts +**/sdist +**/var +**/wheels +**/share/python-wheels +**/*.egg-info +**/.installed.cfg +**/*.egg +**/MANIFEST +**/.DS_Store +**/.env +fly.toml diff --git a/.gitignore b/.gitignore index d23851756..f0feef95d 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,4 @@ share/python-wheels/ MANIFEST .DS_Store .env +fly.toml \ No newline at end of file diff --git a/examples/server/Dockerfile b/Dockerfile similarity index 82% rename from examples/server/Dockerfile rename to Dockerfile index 7acfa2de8..f10db656d 100644 --- a/examples/server/Dockerfile +++ b/Dockerfile @@ -7,13 +7,14 @@ COPY *.py /app COPY pyproject.toml /app COPY src/ /app/src/ +COPY examples/ /app/examples/ WORKDIR /app RUN ls --recursive /app/ RUN pip3 install --upgrade -r requirements.txt RUN python -m build . RUN pip3 install . - +RUN pip3 install gunicorn # If running on Ubuntu, Azure TTS requires some extra config # https://learn.microsoft.com/en-us/azure/ai-services/speech-service/quickstarts/setup-platform?pivots=programming-language-python&tabs=linux%2Cubuntu%2Cdotnetcli%2Cdotnet%2Cjre%2Cmaven%2Cnodejs%2Cmac%2Cpypi @@ -36,4 +37,4 @@ WORKDIR /app EXPOSE 8000 # run -CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"] +CMD ["gunicorn", "--workers=2", "--log-level", "debug", "--chdir", "examples/server", "--capture-output", "daily-bot-manager:app", "--bind=0.0.0.0:8000"] diff --git a/examples/server/daily-bot-manager.py b/examples/server/daily-bot-manager.py index 8d9c1ca02..6bea559e1 100644 --- a/examples/server/daily-bot-manager.py +++ b/examples/server/daily-bot-manager.py @@ -14,10 +14,10 @@ app = Flask(__name__) CORS(app) APPS = { - "chatbot": "examples/starter-apps/chatbot.py", - "patient-intake": "examples/starter-apps/patient-intake.py", - "storybot": "examples/starter-apps/storybot.py", - "translator": "examples/starter-apps/translator.py" + "chatbot": "../starter-apps/chatbot.py", + "patient-intake": "../starter-apps/patient-intake.py", + "storybot": "../starter-apps/storybot.py", + "translator": "../starter-apps/translator.py" } daily_api_key = os.getenv("DAILY_API_KEY") @@ -157,7 +157,7 @@ def start(botname): else: return jsonify({"room_url": room_url, "token": token}) except BaseException as e: - return "There was a problem starting the bot: {e}", 500 + return f"There was a problem starting the bot: {e}", 500 @app.route("/healthz") diff --git a/examples/starter-apps/translator.py b/examples/starter-apps/translator.py index abb1e02b4..5b8c3286b 100644 --- a/examples/starter-apps/translator.py +++ b/examples/starter-apps/translator.py @@ -84,7 +84,6 @@ async def main(room_url: str, token): mic_enabled=True, mic_sample_rate=16000, camera_enabled=False, - vad_enabled=True, ) tts = AzureTTSService( api_key=os.getenv("AZURE_SPEECH_API_KEY"), @@ -98,7 +97,7 @@ async def main(room_url: str, token): tp = TranslationProcessor("Spanish") lfra = LLMFullResponseAggregator() ts = TranslationSubtitles("spanish") - pipeline = Pipeline([sa, tp, llm, lfra, ts]) + pipeline = Pipeline([sa, tp, llm, lfra, ts, tts]) transport.transcription_settings["extra"]["endpointing"] = True transport.transcription_settings["extra"]["punctuate"] = True diff --git a/pyproject.toml b/pyproject.toml index a559076f9..18ca2e56b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,15 +26,14 @@ dependencies = [ "daily-python", "fal", "faster_whisper", + "flask", + "flask_cors", "google-cloud-texttospeech", "numpy", "openai", "Pillow", "pyht", "python-dotenv", - "torch", - "torchaudio", - "pyaudio", "typing-extensions", "websockets" ] diff --git a/src/dailyai/transports/local_transport.py b/src/dailyai/transports/local_transport.py index 1e791a467..4b24fb535 100644 --- a/src/dailyai/transports/local_transport.py +++ b/src/dailyai/transports/local_transport.py @@ -1,7 +1,6 @@ import asyncio import numpy as np import tkinter as tk -import pyaudio from dailyai.transports.threaded_transport import ThreadedTransport @@ -9,6 +8,13 @@ from dailyai.transports.threaded_transport import ThreadedTransport class LocalTransport(ThreadedTransport): def __init__(self, **kwargs): super().__init__(**kwargs) + try: + global pyaudio + import pyaudio + except ModuleNotFoundError as e: + print(f"Exception: {e}") + print("In order to use the local transport, you'll need to `pip install pyaudio`. On MacOS, you'll also need to `brew install portaudio`.") + raise Exception(f"Missing module: {e}") self._sample_width = kwargs.get("sample_width") or 2 self._n_channels = kwargs.get("n_channels") or 1 self._tk_root = kwargs.get("tk_root") or None diff --git a/src/dailyai/transports/threaded_transport.py b/src/dailyai/transports/threaded_transport.py index d529731cd..033cb97a2 100644 --- a/src/dailyai/transports/threaded_transport.py +++ b/src/dailyai/transports/threaded_transport.py @@ -3,8 +3,7 @@ import asyncio import itertools import logging import numpy as np -import pyaudio -import torch + import queue import threading import time @@ -29,22 +28,6 @@ from dailyai.pipeline.pipeline import Pipeline from dailyai.services.ai_services import TTSService from dailyai.transports.abstract_transport import AbstractTransport -torch.set_num_threads(1) - -model, utils = torch.hub.load( - repo_or_dir="snakers4/silero-vad", model="silero_vad", force_reload=False -) - -(get_speech_timestamps, save_audio, read_audio, VADIterator, collect_chunks) = utils - -# Taken from utils_vad.py - - -def validate(model, inputs: torch.Tensor): - with torch.no_grad(): - outs = model(inputs) - return outs - # Provided by Alexander Veysov @@ -58,12 +41,7 @@ def int2float(sound): return sound -FORMAT = pyaudio.paInt16 -CHANNELS = 1 SAMPLE_RATE = 16000 -CHUNK = int(SAMPLE_RATE / 10) - -audio = pyaudio.PyAudio() class VADState(Enum): @@ -90,6 +68,24 @@ class ThreadedTransport(AbstractTransport): "Sorry, you can't use speaker_enabled and vad_enabled at the same time. Please set one to False." ) + if self._vad_enabled: + try: + global torch, torchaudio + import torch + # We don't use torchaudio here, but we need to try importing it because + # Silero uses it + import torchaudio + torch.set_num_threads(1) + + (self.model, self.utils) = torch.hub.load( + repo_or_dir="snakers4/silero-vad", model="silero_vad", force_reload=False + ) + + except ModuleNotFoundError as e: + print(f"Exception: {e}") + print("In order to use VAD, you'll need to install the `torch` and `torchaudio` modules.") + raise Exception(f"Missing module(s): {e}") + self._vad_samples = 1536 vad_frame_s = self._vad_samples / SAMPLE_RATE self._vad_start_frames = round(self._vad_start_s / vad_frame_s) @@ -276,7 +272,7 @@ class ThreadedTransport(AbstractTransport): audio_chunk = self.read_audio_frames(self._vad_samples) audio_int16 = np.frombuffer(audio_chunk, np.int16) audio_float32 = int2float(audio_int16) - new_confidence = model( + new_confidence = self.model( torch.from_numpy(audio_float32), 16000).item() speaking = new_confidence > 0.5