Improved the accuracy of the UserBotLatencyObserver and UserBotLatencyLogObserver

changelog/3637.changed.5.md

@@ -0,0 +1 @@
+- Improved the accuracy of the `UserBotLatencyObserver` and `UserBotLatencyLogObserver` by measuring from the time when the user actually starts speaking.

src/pipecat/observers/loggers/user_bot_latency_log_observer.py

@@ -79,7 +79,7 @@ class UserBotLatencyLogObserver(BaseObserver):
         if isinstance(data.frame, VADUserStartedSpeakingFrame):
             self._user_stopped_time = 0
         elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
-            self._user_stopped_time = data.frame.timestamp
+            self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
         elif isinstance(data.frame, (EndFrame, CancelFrame)):
             self._log_summary()
         elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time:

src/pipecat/observers/user_bot_latency_observer.py

@@ -72,8 +72,10 @@ class UserBotLatencyObserver(BaseObserver):
             # Reset when user starts speaking
             self._user_stopped_time = None
         elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
-            # Record timestamp when user stops speaking
-            self._user_stopped_time = data.frame.timestamp
+            # Record the actual time the user stopped speaking, which is
+            # the VAD determination time minus the stop_secs silence duration
+            # that had to elapse before the VAD confirmed speech ended.
+            self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
         elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time:
             # Calculate and emit latency
             latency = time.time() - self._user_stopped_time