VAD fallback (#97)

* Silero VAD preferred with webrtc fallback

* webrtc VAD neds a different sample size

* fixup

* fixup
This commit is contained in:
chadbailey59
2024-04-04 13:31:07 -05:00
committed by GitHub
parent 385b51ac83
commit 03ea208361
2 changed files with 59 additions and 21 deletions

View File

@@ -5,6 +5,7 @@ import signal
import threading
import types
from enum import Enum
from functools import partial
from typing import Any
@@ -33,6 +34,11 @@ except ModuleNotFoundError as e:
from dailyai.transports.threaded_transport import ThreadedTransport
NUM_CHANNELS = 1
SPEECH_THRESHOLD = 0.90
VAD_RESET_PERIOD_MS = 2000
class DailyTransport(ThreadedTransport, EventHandler):
_daily_initialized = False
@@ -55,6 +61,7 @@ class DailyTransport(ThreadedTransport, EventHandler):
start_transcription: bool = False,
**kwargs,
):
kwargs['has_webrtc_vad'] = True
# This will call ThreadedTransport.__init__ method, not EventHandler
super().__init__(**kwargs)
@@ -86,6 +93,12 @@ class DailyTransport(ThreadedTransport, EventHandler):
self._event_handlers = {}
self.webrtc_vad = Daily.create_native_vad(
reset_period_ms=VAD_RESET_PERIOD_MS,
sample_rate=self._speaker_sample_rate,
channels=NUM_CHANNELS
)
def _patch_method(self, event_name, *args, **kwargs):
try:
for handler in self._event_handlers[event_name]:
@@ -106,6 +119,18 @@ class DailyTransport(ThreadedTransport, EventHandler):
self._logger.error(f"Exception in event handler {event_name}: {e}")
raise e
def _webrtc_vad_analyze(self):
buffer = self.read_audio_frames(
int(self._vad_samples))
if len(buffer) > 0:
confidence = self.webrtc_vad.analyze_frames(buffer)
# yeses = int(confidence * 20.0)
# nos = 20 - yeses
# out = "!" * yeses + "." * nos
# print(f"!!! confidence: {out} {confidence}")
talking = confidence > SPEECH_THRESHOLD
return talking
def add_event_handler(self, event_name: str, handler):
if not event_name.startswith("on_"):
raise Exception(