From d5644c3ab9be00dcf3a3143c5b24a0cc5ba35419 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= Date: Wed, 22 May 2024 15:26:32 -0700 Subject: [PATCH] vad: re-add volume exponential smoothing --- CHANGELOG.md | 7 +++++++ src/pipecat/services/ai_services.py | 14 ++++---------- src/pipecat/vad/vad_analyzer.py | 13 +++++++++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8798bc9aa..cccd516bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to **pipecat** will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Fixed + +- Re-add exponential smoothing after volume calculation. This makes sure the + volume value being used doesn't fluctuate so much. + ## [0.0.20] - 2024-05-22 ### Added diff --git a/src/pipecat/services/ai_services.py b/src/pipecat/services/ai_services.py index f42086dad..ca2e86354 100644 --- a/src/pipecat/services/ai_services.py +++ b/src/pipecat/services/ai_services.py @@ -4,9 +4,7 @@ # SPDX-License-Identifier: BSD 2-Clause License # -import array import io -import math import wave from abc import abstractmethod @@ -111,7 +109,7 @@ class STTService(AIService): (self._content, self._wave) = self._new_wave() self._silence_num_frames = 0 # Volume exponential smoothing - self._smoothing_factor = 0.5 + self._smoothing_factor = 0.4 self._prev_volume = 1 - self._smoothing_factor @abstractmethod @@ -127,17 +125,13 @@ class STTService(AIService): ww.setframerate(self._sample_rate) return (content, ww) - def _get_smoothed_volume( - self, - frame: AudioRawFrame, - prev_volume: float, - factor: float) -> float: + def _get_smoothed_volume(self, frame: AudioRawFrame) -> float: volume = calculate_audio_volume(frame.audio, frame.sample_rate) - return exp_smoothing(volume, prev_volume, factor) + return exp_smoothing(volume, self._prev_volume, self._smoothing_factor) async def _append_audio(self, frame: AudioRawFrame): # Try to filter out empty background noise - volume = self._get_smoothed_volume(frame, self._prev_volume, self._smoothing_factor) + volume = self._get_smoothed_volume(frame) if volume >= self._min_volume: # If volume is high enough, write new data to wave file self._wave.writeframes(frame.audio) diff --git a/src/pipecat/vad/vad_analyzer.py b/src/pipecat/vad/vad_analyzer.py index d05b85f9b..cd49d282b 100644 --- a/src/pipecat/vad/vad_analyzer.py +++ b/src/pipecat/vad/vad_analyzer.py @@ -9,7 +9,7 @@ from enum import Enum from pydantic.main import BaseModel -from pipecat.utils.audio import calculate_audio_volume +from pipecat.utils.audio import calculate_audio_volume, exp_smoothing class VADState(Enum): @@ -45,6 +45,10 @@ class VADAnalyzer: self._vad_buffer = b"" + # Volume exponential smoothing + self._smoothing_factor = 0.4 + self._prev_volume = 1 - self._smoothing_factor + @property def sample_rate(self): return self._sample_rate @@ -57,6 +61,10 @@ class VADAnalyzer: def voice_confidence(self, buffer) -> float: pass + def _get_smoothed_volume(self, audio: bytes) -> float: + volume = calculate_audio_volume(audio, self._sample_rate) + return exp_smoothing(volume, self._prev_volume, self._smoothing_factor) + def analyze_audio(self, buffer) -> VADState: self._vad_buffer += buffer @@ -69,7 +77,8 @@ class VADAnalyzer: confidence = self.voice_confidence(audio_frames) - volume = calculate_audio_volume(audio_frames, self._sample_rate) + volume = self._get_smoothed_volume(audio_frames) + self._prev_volume = volume speaking = confidence >= self._params.confidence and volume >= self._params.min_volume