diff --git a/CHANGELOG.md b/CHANGELOG.md index 251539ba7..741d86c4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `VADUserStartedSpeakingFrame` and `VADUserStoppedSpeakingFrame`, + indicating when the VAD detected the user to start and stop speaking. These + events are helpful when using smart turn detection, as the user's stop time + can differ from when their turn ends (signified by UserStoppedSpeakingFrame). + - Added `TranslationFrame`, a new frame type that contains a translated transcription. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index 073112a9f..0d7d90d78 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -587,6 +587,20 @@ class EmulateUserStoppedSpeakingFrame(SystemFrame): pass +@dataclass +class VADUserStartedSpeakingFrame(SystemFrame): + """Frame emitted when VAD detects the user has definitively started speaking.""" + + pass + + +@dataclass +class VADUserStoppedSpeakingFrame(SystemFrame): + """Frame emitted when VAD detects the user has definitively stopped speaking.""" + + pass + + @dataclass class BotInterruptionFrame(SystemFrame): """Emitted by when the bot should be interrupted. This will mainly cause the diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 497b21cda..acb0058c6 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -32,6 +32,8 @@ from pipecat.frames.frames import ( UserStartedSpeakingFrame, UserStoppedSpeakingFrame, VADParamsUpdateFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, ) from pipecat.metrics.metrics import MetricsData from pipecat.processors.frame_processor import FrameDirection, FrameProcessor @@ -248,10 +250,13 @@ class BaseInputTransport(FrameProcessor): self._params.turn_analyzer is None or not self._params.turn_analyzer.speech_triggered ) - if can_create_user_frames: - if new_vad_state == VADState.SPEAKING: + if new_vad_state == VADState.SPEAKING: + await self.push_frame(VADUserStartedSpeakingFrame()) + if can_create_user_frames: frame = UserStartedSpeakingFrame() - elif new_vad_state == VADState.QUIET: + elif new_vad_state == VADState.QUIET: + await self.push_frame(VADUserStoppedSpeakingFrame()) + if can_create_user_frames: frame = UserStoppedSpeakingFrame() if frame: