introduce and push UserSpeakingFrame upstream/downstream

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+- Added `UserSpeakingFrame`. This will be sent upstream and downstream while VAD
+  detects the user is speaking.
+
 - Expanded support for universal `LLMContext` to more LLM services. Using the
   universal `LLMContext` and associated `LLMContextAggregatorPair` is a
   pre-requisite for using `LLMSwitcher` to switch between LLMs at runtime.

src/pipecat/frames/frames.py

@@ -898,6 +898,16 @@ class UserStoppedSpeakingFrame(SystemFrame):
     emulated: bool = False
 
 
+@dataclass
+class UserSpeakingFrame(SystemFrame):
+    """Frame indicating the user is speaking.
+
+    Emitted by VAD to indicate the user is speaking.
+    """
+
+    pass
+
+
 @dataclass
 class EmulateUserStartedSpeakingFrame(SystemFrame):
     """Frame to emulate user started speaking behavior.

src/pipecat/pipeline/task.py

@@ -32,15 +32,11 @@ from pipecat.frames.frames import (
     Frame,
     HeartbeatFrame,
     InputAudioRawFrame,
-    InterimTranscriptionFrame,
-    LLMFullResponseEndFrame,
     MetricsFrame,
     StartFrame,
     StopFrame,
     StopTaskFrame,
-    TranscriptionFrame,
+    UserSpeakingFrame,
-    UserStartedSpeakingFrame,
-    UserStoppedSpeakingFrame,
 )
 from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData
 from pipecat.observers.base_observer import BaseObserver
@@ -145,14 +141,7 @@ class PipelineTask(BasePipelineTask):
         conversation_id: Optional[str] = None,
         enable_tracing: bool = False,
         enable_turn_tracking: bool = True,
-        idle_timeout_frames: Tuple[Type[Frame], ...] = (
+        idle_timeout_frames: Tuple[Type[Frame], ...] = (BotSpeakingFrame, UserSpeakingFrame),
-            BotSpeakingFrame,
-            InterimTranscriptionFrame,
-            LLMFullResponseEndFrame,
-            TranscriptionFrame,
-            UserStartedSpeakingFrame,
-            UserStoppedSpeakingFrame,
-        ),
         idle_timeout_secs: Optional[float] = IDLE_TIMEOUT_SECS,
         observers: Optional[List[BaseObserver]] = None,
         task_manager: Optional[BaseTaskManager] = None,

src/pipecat/transports/base_input.py

@@ -39,6 +39,7 @@ from pipecat.frames.frames import (
     StartInterruptionFrame,
     StopFrame,
     SystemFrame,
+    UserSpeakingFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
     VADParamsUpdateFrame,
@@ -411,7 +412,7 @@ class BaseInputTransport(FrameProcessor):
             )
         return state
 
-    async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState):
+    async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState) -> VADState:
         """Handle Voice Activity Detection results and generate appropriate frames."""
         new_vad_state = await self._vad_analyze(audio_frame)
         if (
@@ -489,6 +490,10 @@ class BaseInputTransport(FrameProcessor):
                 if self._params.turn_analyzer:
                     await self._run_turn_analyzer(frame, vad_state, previous_vad_state)
 
+                if vad_state == VADState.SPEAKING:
+                    await self.push_frame(UserSpeakingFrame())
+                    await self.push_frame(UserSpeakingFrame(), FrameDirection.UPSTREAM)
+
                 # Push audio downstream if passthrough is set.
                 if self._params.audio_in_passthrough:
                     await self.push_frame(frame)