diff --git a/changelog/3637.changed.5.md b/changelog/3637.changed.5.md
new file mode 100644
index 000000000..367f9f6e3
--- /dev/null
+++ b/changelog/3637.changed.5.md
@@ -0,0 +1 @@
+- Improved the accuracy of the `UserBotLatencyObserver` and `UserBotLatencyLogObserver` by measuring from the time when the user actually starts speaking.
\ No newline at end of file
diff --git a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py
index b15f989d1..2323a36ee 100644
--- a/src/pipecat/observers/loggers/user_bot_latency_log_observer.py
+++ b/src/pipecat/observers/loggers/user_bot_latency_log_observer.py
@@ -79,7 +79,7 @@ class UserBotLatencyLogObserver(BaseObserver):
         if isinstance(data.frame, VADUserStartedSpeakingFrame):
             self._user_stopped_time = 0
         elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
-            self._user_stopped_time = data.frame.timestamp
+            self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
         elif isinstance(data.frame, (EndFrame, CancelFrame)):
             self._log_summary()
         elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time:
diff --git a/src/pipecat/observers/user_bot_latency_observer.py b/src/pipecat/observers/user_bot_latency_observer.py
index 17612ca76..37d5bc1a0 100644
--- a/src/pipecat/observers/user_bot_latency_observer.py
+++ b/src/pipecat/observers/user_bot_latency_observer.py
@@ -72,8 +72,10 @@ class UserBotLatencyObserver(BaseObserver):
             # Reset when user starts speaking
             self._user_stopped_time = None
         elif isinstance(data.frame, VADUserStoppedSpeakingFrame):
-            # Record timestamp when user stops speaking
-            self._user_stopped_time = data.frame.timestamp
+            # Record the actual time the user stopped speaking, which is
+            # the VAD determination time minus the stop_secs silence duration
+            # that had to elapse before the VAD confirmed speech ended.
+            self._user_stopped_time = data.frame.timestamp - data.frame.stop_secs
         elif isinstance(data.frame, BotStartedSpeakingFrame) and self._user_stopped_time:
             # Calculate and emit latency
             latency = time.time() - self._user_stopped_time