diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cedaa5d4..db7458c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added `UserSpeakingFrame`. This will be sent upstream and downstream while VAD + detects the user is speaking. + - Expanded support for universal `LLMContext` to more LLM services. Using the universal `LLMContext` and associated `LLMContextAggregatorPair` is a pre-requisite for using `LLMSwitcher` to switch between LLMs at runtime. diff --git a/src/pipecat/frames/frames.py b/src/pipecat/frames/frames.py index ac939f0fa..0d8471756 100644 --- a/src/pipecat/frames/frames.py +++ b/src/pipecat/frames/frames.py @@ -898,6 +898,16 @@ class UserStoppedSpeakingFrame(SystemFrame): emulated: bool = False +@dataclass +class UserSpeakingFrame(SystemFrame): + """Frame indicating the user is speaking. + + Emitted by VAD to indicate the user is speaking. + """ + + pass + + @dataclass class EmulateUserStartedSpeakingFrame(SystemFrame): """Frame to emulate user started speaking behavior. diff --git a/src/pipecat/pipeline/task.py b/src/pipecat/pipeline/task.py index 46c6b8312..8c1e4c044 100644 --- a/src/pipecat/pipeline/task.py +++ b/src/pipecat/pipeline/task.py @@ -32,15 +32,11 @@ from pipecat.frames.frames import ( Frame, HeartbeatFrame, InputAudioRawFrame, - InterimTranscriptionFrame, - LLMFullResponseEndFrame, MetricsFrame, StartFrame, StopFrame, StopTaskFrame, - TranscriptionFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, + UserSpeakingFrame, ) from pipecat.metrics.metrics import ProcessingMetricsData, TTFBMetricsData from pipecat.observers.base_observer import BaseObserver @@ -145,14 +141,7 @@ class PipelineTask(BasePipelineTask): conversation_id: Optional[str] = None, enable_tracing: bool = False, enable_turn_tracking: bool = True, - idle_timeout_frames: Tuple[Type[Frame], ...] = ( - BotSpeakingFrame, - InterimTranscriptionFrame, - LLMFullResponseEndFrame, - TranscriptionFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, - ), + idle_timeout_frames: Tuple[Type[Frame], ...] = (BotSpeakingFrame, UserSpeakingFrame), idle_timeout_secs: Optional[float] = IDLE_TIMEOUT_SECS, observers: Optional[List[BaseObserver]] = None, task_manager: Optional[BaseTaskManager] = None, diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 2052eed00..865619879 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -39,6 +39,7 @@ from pipecat.frames.frames import ( StartInterruptionFrame, StopFrame, SystemFrame, + UserSpeakingFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, VADParamsUpdateFrame, @@ -411,7 +412,7 @@ class BaseInputTransport(FrameProcessor): ) return state - async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState): + async def _handle_vad(self, audio_frame: InputAudioRawFrame, vad_state: VADState) -> VADState: """Handle Voice Activity Detection results and generate appropriate frames.""" new_vad_state = await self._vad_analyze(audio_frame) if ( @@ -489,6 +490,10 @@ class BaseInputTransport(FrameProcessor): if self._params.turn_analyzer: await self._run_turn_analyzer(frame, vad_state, previous_vad_state) + if vad_state == VADState.SPEAKING: + await self.push_frame(UserSpeakingFrame()) + await self.push_frame(UserSpeakingFrame(), FrameDirection.UPSTREAM) + # Push audio downstream if passthrough is set. if self._params.audio_in_passthrough: await self.push_frame(frame)