Merge pull request #164 from pipecat-ai/readd-vad-exp-smoothing

vad: re-add volume exponential smoothing

CHANGELOG.md

@@ -5,6 +5,13 @@ All notable changes to **pipecat** will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Fixed
+
+- Re-add exponential smoothing after volume calculation. This makes sure the
+  volume value being used doesn't fluctuate so much.
+
 ## [0.0.20] - 2024-05-22
 
 ### Added

src/pipecat/services/ai_services.py

@@ -4,9 +4,7 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import array
 import io
-import math
 import wave
 
 from abc import abstractmethod
@@ -111,7 +109,7 @@ class STTService(AIService):
         (self._content, self._wave) = self._new_wave()
         self._silence_num_frames = 0
         # Volume exponential smoothing
-        self._smoothing_factor = 0.5
+        self._smoothing_factor = 0.4
         self._prev_volume = 1 - self._smoothing_factor
 
     @abstractmethod
@@ -127,17 +125,13 @@ class STTService(AIService):
         ww.setframerate(self._sample_rate)
         return (content, ww)
 
-    def _get_smoothed_volume(
-            self,
-            frame: AudioRawFrame,
-            prev_volume: float,
-            factor: float) -> float:
+    def _get_smoothed_volume(self, frame: AudioRawFrame) -> float:
         volume = calculate_audio_volume(frame.audio, frame.sample_rate)
-        return exp_smoothing(volume, prev_volume, factor)
+        return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
 
     async def _append_audio(self, frame: AudioRawFrame):
         # Try to filter out empty background noise
-        volume = self._get_smoothed_volume(frame, self._prev_volume, self._smoothing_factor)
+        volume = self._get_smoothed_volume(frame)
         if volume >= self._min_volume:
             # If volume is high enough, write new data to wave file
             self._wave.writeframes(frame.audio)

src/pipecat/vad/vad_analyzer.py

@@ -9,7 +9,7 @@ from enum import Enum
 
 from pydantic.main import BaseModel
 
-from pipecat.utils.audio import calculate_audio_volume
+from pipecat.utils.audio import calculate_audio_volume, exp_smoothing
 
 
 class VADState(Enum):
@@ -45,6 +45,10 @@ class VADAnalyzer:
 
         self._vad_buffer = b""
 
+        # Volume exponential smoothing
+        self._smoothing_factor = 0.4
+        self._prev_volume = 1 - self._smoothing_factor
+
     @property
     def sample_rate(self):
         return self._sample_rate
@@ -57,6 +61,10 @@ class VADAnalyzer:
     def voice_confidence(self, buffer) -> float:
         pass
 
+    def _get_smoothed_volume(self, audio: bytes) -> float:
+        volume = calculate_audio_volume(audio, self._sample_rate)
+        return exp_smoothing(volume, self._prev_volume, self._smoothing_factor)
+
     def analyze_audio(self, buffer) -> VADState:
         self._vad_buffer += buffer
 
@@ -69,7 +77,8 @@ class VADAnalyzer:
 
         confidence = self.voice_confidence(audio_frames)
 
-        volume = calculate_audio_volume(audio_frames, self._sample_rate)
+        volume = self._get_smoothed_volume(audio_frames)
+        self._prev_volume = volume
 
         speaking = confidence >= self._params.confidence and volume >= self._params.min_volume