Merge pull request #1666 from pipecat-ai/mb/add-vad-start-stop-events

Add VADUserStartedSpeakingFrame and VADUserStoppedSpeakingFrame

CHANGELOG.md

@@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added `VADUserStartedSpeakingFrame` and `VADUserStoppedSpeakingFrame`,
+  indicating when the VAD detected the user to start and stop speaking. These
+  events are helpful when using smart turn detection, as the user's stop time
+  can differ from when their turn ends (signified by UserStoppedSpeakingFrame).
+
 - Added `TranslationFrame`, a new frame type that contains a translated
   transcription.
 

src/pipecat/frames/frames.py

@@ -587,6 +587,20 @@ class EmulateUserStoppedSpeakingFrame(SystemFrame):
     pass
 
 
+@dataclass
+class VADUserStartedSpeakingFrame(SystemFrame):
+    """Frame emitted when VAD detects the user has definitively started speaking."""
+
+    pass
+
+
+@dataclass
+class VADUserStoppedSpeakingFrame(SystemFrame):
+    """Frame emitted when VAD detects the user has definitively stopped speaking."""
+
+    pass
+
+
 @dataclass
 class BotInterruptionFrame(SystemFrame):
     """Emitted by when the bot should be interrupted. This will mainly cause the

src/pipecat/transports/base_input.py

@@ -32,6 +32,8 @@ from pipecat.frames.frames import (
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
     VADParamsUpdateFrame,
+    VADUserStartedSpeakingFrame,
+    VADUserStoppedSpeakingFrame,
 )
 from pipecat.metrics.metrics import MetricsData
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
@@ -248,10 +250,13 @@ class BaseInputTransport(FrameProcessor):
                 self._params.turn_analyzer is None
                 or not self._params.turn_analyzer.speech_triggered
             )
-            if can_create_user_frames:
+            if new_vad_state == VADState.SPEAKING:
-                if new_vad_state == VADState.SPEAKING:
+                await self.push_frame(VADUserStartedSpeakingFrame())
+                if can_create_user_frames:
                     frame = UserStartedSpeakingFrame()
-                elif new_vad_state == VADState.QUIET:
+            elif new_vad_state == VADState.QUIET:
+                await self.push_frame(VADUserStoppedSpeakingFrame())
+                if can_create_user_frames:
                     frame = UserStoppedSpeakingFrame()
 
             if frame: