diff --git a/src/pipecat/extensions/voicemail/voicemail_detector.py b/src/pipecat/extensions/voicemail/voicemail_detector.py
index 47a2e5c46..52b7feeab 100644
--- a/src/pipecat/extensions/voicemail/voicemail_detector.py
+++ b/src/pipecat/extensions/voicemail/voicemail_detector.py
@@ -280,11 +280,13 @@ class ClassificationProcessor(FrameProcessor):
 
         elif isinstance(frame, UserStartedSpeakingFrame):
             # User started speaking - set the voicemail event
-            self._voicemail_event.set()
+            if self._voicemail_detected:
+                self._voicemail_event.set()
 
         elif isinstance(frame, UserStoppedSpeakingFrame):
             # User stopped speaking - clear the voicemail event
-            self._voicemail_event.clear()
+            if self._voicemail_detected:
+                self._voicemail_event.clear()
 
         else:
             # Pass all non-LLM frames through