From 3e13678f238810e32d06dd3ea443cfef8246834f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Fri, 17 May 2024 17:13:31 -0700
Subject: [PATCH] vad: use exponential smoothed volume to improve speech
 detection

---
 CHANGELOG.md                        |  6 +++++-
 src/pipecat/services/ai_services.py |  6 +++---
 src/pipecat/vad/vad_analyzer.py     | 31 ++++++++++++++++++-----------
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f80a6cf3d..69c9e9091 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,10 +13,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Added `VADParams` so you can control voice confidence level and others.
 
-- `VADAnalyzer` now uses an exponential smoothing to avoid sudden changes.
+- `VADAnalyzer` now uses an exponential smoothed volume to improve speech
+  detection. This is useful when voice confidence is high (because there's
+  someone talking near you) but volume is low.
 
 ### Fixed
 
+- Fixed an issue where TTSService was not pushing TextFrames downstream.
+
 - Fixed issues with Ctrl-C program termination.
 
 - Fixed an issue that was causing `StopTaskFrame` to actually not exit the
diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py
index 696c61a73..bb697a84f 100644
--- a/src/pipecat/services/ai_services.py
+++ b/src/pipecat/services/ai_services.py
@@ -94,7 +94,7 @@ class STTService(AIService):
     """STTService is a base class for speech-to-text services."""
 
     def __init__(self,
-                 min_rms: int = 75,
+                 min_rms: int = 100,
                  max_silence_secs: float = 0.3,
                  max_buffer_secs: float = 1.5,
                  sample_rate: int = 16000,
@@ -107,8 +107,8 @@ class STTService(AIService):
         self._num_channels = num_channels
         (self._content, self._wave) = self._new_wave()
         self._silence_num_frames = 0
-        # Exponential smoothing
-        self._smoothing_factor = 0.08
+        # Volume exponential smoothing
+        self._smoothing_factor = 0.5
         self._prev_rms = 1 - self._smoothing_factor
 
     @abstractmethod
diff --git a/src/pipecat/vad/vad_analyzer.py b/src/pipecat/vad/vad_analyzer.py
index 1d68e2af6..15f036387 100644
--- a/src/pipecat/vad/vad_analyzer.py
+++ b/src/pipecat/vad/vad_analyzer.py
@@ -4,6 +4,9 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
+import array
+import math
+
 from abc import abstractmethod
 from enum import Enum
 
@@ -20,9 +23,10 @@ class VADState(Enum):
 
 
 class VADParams(BaseModel):
-    confidence: float = 0.5
+    confidence: float = 0.6
     start_secs: float = 0.2
     stop_secs: float = 0.8
+    min_rms: int = 1000
 
 
 class VADAnalyzer:
@@ -43,9 +47,9 @@ class VADAnalyzer:
 
         self._vad_buffer = b""
 
-        # Exponential smoothing
-        self._smoothing_factor = 0.6
-        self._prev_confidence = 1 - self._smoothing_factor
+        # Volume exponential smoothing
+        self._smoothing_factor = 0.5
+        self._prev_rms = 1 - self._smoothing_factor
 
     @property
     def sample_rate(self):
@@ -59,10 +63,13 @@ class VADAnalyzer:
     def voice_confidence(self, buffer) -> float:
         pass
 
-    def _smoothed_confidence(self, audio_frames, prev_confidence, factor):
-        confidence = self.voice_confidence(audio_frames)
-        smoothed = exp_smoothing(confidence, prev_confidence, factor)
-        return smoothed
+    def _get_smoothed_volume(self, audio: bytes, prev_rms: float, factor: float) -> float:
+        # https://docs.python.org/3/library/array.html
+        audio_array = array.array('h', audio)
+        squares = [sample**2 for sample in audio_array]
+        mean = sum(squares) / len(audio_array)
+        rms = math.sqrt(mean)
+        return exp_smoothing(rms, prev_rms, factor)
 
     def analyze_audio(self, buffer) -> VADState:
         self._vad_buffer += buffer
@@ -74,11 +81,11 @@ class VADAnalyzer:
         audio_frames = self._vad_buffer[:num_required_bytes]
         self._vad_buffer = self._vad_buffer[num_required_bytes:]
 
-        confidence = self._smoothed_confidence(
-            audio_frames, self._prev_confidence, self._smoothing_factor)
-        self._prev_confidence = confidence
+        confidence = self.voice_confidence(audio_frames)
+        rms = self._get_smoothed_volume(audio_frames, self._prev_rms, self._smoothing_factor)
+        self._prev_rms = rms
 
-        speaking = confidence >= self._params.confidence
+        speaking = confidence >= self._params.confidence and rms >= self._params.min_rms
 
         if speaking:
             match self._vad_state: