From af1c7d0023b37bd3c9bcbb68c308d5b713125b74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aleix=20Conchillo=20Flaqu=C3=A9?= <aleix@daily.co>
Date: Tue, 21 May 2024 16:26:08 -0700
Subject: [PATCH] interruptions: introduce pyloudnorm to compute loudness

https://github.com/csteinmetz1/pyloudnorm
---
 pyproject.toml                  |  1 +
 src/pipecat/utils/audio.py      | 35 +++++++++++++++++++++++++++++++++
 src/pipecat/vad/vad_analyzer.py | 26 ++++++------------------
 3 files changed, 42 insertions(+), 20 deletions(-)
 create mode 100644 src/pipecat/utils/audio.py

diff --git a/pyproject.toml b/pyproject.toml
index 983a7b4c4..00fbc826b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,6 +24,7 @@ dependencies = [
     "numpy~=1.26.4",
     "loguru~=0.7.0",
     "Pillow~=10.3.0",
+    "pyloudnorm~=0.1.1",
     "typing-extensions~=4.11.0",
 ]
 
diff --git a/src/pipecat/utils/audio.py b/src/pipecat/utils/audio.py
new file mode 100644
index 000000000..ad0813ecd
--- /dev/null
+++ b/src/pipecat/utils/audio.py
@@ -0,0 +1,35 @@
+#
+# Copyright (c) 2024, Daily
+#
+# SPDX-License-Identifier: BSD 2-Clause License
+#
+
+import numpy as np
+import pyloudnorm as pyln
+
+
+def compute_rms(audio: np.ndarray):
+    return np.sqrt(np.mean(audio**2))
+
+
+def normalize_value(value, min_value, max_value):
+    return (value - min_value) / (max_value - min_value)
+
+
+def calculate_audio_volume(audio: bytes, sample_rate: int) -> float:
+    audio_np = np.frombuffer(audio, dtype=np.int16)
+    audio_float = audio_np.astype(np.float64)
+
+    block_size = audio_np.size / sample_rate
+    meter = pyln.Meter(sample_rate, block_size=block_size)
+    loudness = meter.integrated_loudness(audio_float)
+
+    # Loudness goes from -20 to 80 (more or less), where -20 is quiet and 80 is
+    # loud.
+    loudness = normalize_value(loudness, -20, 80)
+
+    return loudness
+
+
+def exp_smoothing(value: float, prev_value: float, factor: float) -> float:
+    return prev_value + factor * (value - prev_value)
diff --git a/src/pipecat/vad/vad_analyzer.py b/src/pipecat/vad/vad_analyzer.py
index 15f036387..8a5eeb098 100644
--- a/src/pipecat/vad/vad_analyzer.py
+++ b/src/pipecat/vad/vad_analyzer.py
@@ -4,15 +4,12 @@
 # SPDX-License-Identifier: BSD 2-Clause License
 #
 
-import array
-import math
-
 from abc import abstractmethod
 from enum import Enum
 
 from pydantic.main import BaseModel
 
-from pipecat.utils.utils import exp_smoothing
+from pipecat.utils.audio import calculate_audio_volume
 
 
 class VADState(Enum):
@@ -26,13 +23,14 @@ class VADParams(BaseModel):
     confidence: float = 0.6
     start_secs: float = 0.2
     stop_secs: float = 0.8
-    min_rms: int = 1000
+    min_volume: float = 0.7
 
 
 class VADAnalyzer:
 
     def __init__(self, sample_rate: int, num_channels: int, params: VADParams):
         self._sample_rate = sample_rate
+        self._num_channels = num_channels
         self._params = params
         self._vad_frames = self.num_frames_required()
         self._vad_frames_num_bytes = self._vad_frames * num_channels * 2
@@ -47,10 +45,6 @@ class VADAnalyzer:
 
         self._vad_buffer = b""
 
-        # Volume exponential smoothing
-        self._smoothing_factor = 0.5
-        self._prev_rms = 1 - self._smoothing_factor
-
     @property
     def sample_rate(self):
         return self._sample_rate
@@ -63,14 +57,6 @@ class VADAnalyzer:
     def voice_confidence(self, buffer) -> float:
         pass
 
-    def _get_smoothed_volume(self, audio: bytes, prev_rms: float, factor: float) -> float:
-        # https://docs.python.org/3/library/array.html
-        audio_array = array.array('h', audio)
-        squares = [sample**2 for sample in audio_array]
-        mean = sum(squares) / len(audio_array)
-        rms = math.sqrt(mean)
-        return exp_smoothing(rms, prev_rms, factor)
-
     def analyze_audio(self, buffer) -> VADState:
         self._vad_buffer += buffer
 
@@ -82,10 +68,10 @@ class VADAnalyzer:
         self._vad_buffer = self._vad_buffer[num_required_bytes:]
 
         confidence = self.voice_confidence(audio_frames)
-        rms = self._get_smoothed_volume(audio_frames, self._prev_rms, self._smoothing_factor)
-        self._prev_rms = rms
 
-        speaking = confidence >= self._params.confidence and rms >= self._params.min_rms
+        volume = calculate_audio_volume(audio_frames, self._sample_rate)
+
+        speaking = confidence >= self._params.confidence and volume >= self._params.min_volume
 
         if speaking:
             match self._vad_state: