diff --git a/changelog/3628.fixed.md b/changelog/3628.fixed.md new file mode 100644 index 000000000..69a00c30e --- /dev/null +++ b/changelog/3628.fixed.md @@ -0,0 +1 @@ +- Fixed `VADController` not broadcasting `SpeechControlParamsFrame` on startup, which prevented STT services from receiving VAD params needed for TTFB measurement. diff --git a/src/pipecat/audio/vad/vad_controller.py b/src/pipecat/audio/vad/vad_controller.py index 66f7199e5..1db590e10 100644 --- a/src/pipecat/audio/vad/vad_controller.py +++ b/src/pipecat/audio/vad/vad_controller.py @@ -106,6 +106,8 @@ class VADController(BaseObject): async def _start(self, frame: StartFrame): self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate) + # Broadcast initial VAD params so other services (e.g. STT) can use them + await self.broadcast_frame(SpeechControlParamsFrame, vad_params=self._vad_analyzer.params) async def _handle_audio(self, frame: InputAudioRawFrame): """Process an audio chunk and emit speech events as needed. diff --git a/src/pipecat/processors/audio/vad_processor.py b/src/pipecat/processors/audio/vad_processor.py index 074ea57e6..824164b58 100644 --- a/src/pipecat/processors/audio/vad_processor.py +++ b/src/pipecat/processors/audio/vad_processor.py @@ -93,8 +93,10 @@ class VADProcessor(FrameProcessor): """ await super().process_frame(frame, direction) + # Forward the frame first, then let VAD controller process. This ensures: + # 1. StartFrame reaches downstream before SpeechControlParamsFrame is broadcast + # 2. Audio flows through immediately while VAD detection happens after + await self.push_frame(frame, direction) + # Let the VAD controller handle the frame await self._vad_controller.process_frame(frame) - - # Always forward the frame - await self.push_frame(frame, direction) diff --git a/tests/test_vad_controller.py b/tests/test_vad_controller.py index b649a906f..c208b5122 100644 --- a/tests/test_vad_controller.py +++ b/tests/test_vad_controller.py @@ -7,9 +7,9 @@ import unittest from typing import List -from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState +from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams, VADState from pipecat.audio.vad.vad_controller import VADController -from pipecat.frames.frames import Frame, InputAudioRawFrame, StartFrame +from pipecat.frames.frames import Frame, InputAudioRawFrame, SpeechControlParamsFrame, StartFrame from pipecat.processors.frame_processor import FrameDirection @@ -185,6 +185,26 @@ class TestVADController(unittest.IsolatedAsyncioTestCase): await controller.process_frame(audio_frame) self.assertEqual(events_triggered, []) + async def test_start_frame_broadcasts_vad_params(self): + """Test that StartFrame triggers broadcast of SpeechControlParamsFrame with VAD params.""" + analyzer = MockVADAnalyzer() + controller = VADController(analyzer) + + broadcast_calls: List[tuple] = [] + + @controller.event_handler("on_broadcast_frame") + async def on_broadcast_frame(_controller, frame_cls, **kwargs): + broadcast_calls.append((frame_cls, kwargs)) + + start_frame = StartFrame(audio_in_sample_rate=16000, audio_out_sample_rate=16000) + await controller.process_frame(start_frame) + + # Should have broadcast SpeechControlParamsFrame with VAD params + self.assertEqual(len(broadcast_calls), 1) + self.assertEqual(broadcast_calls[0][0], SpeechControlParamsFrame) + self.assertIn("vad_params", broadcast_calls[0][1]) + self.assertIsInstance(broadcast_calls[0][1]["vad_params"], VADParams) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_vad_processor.py b/tests/test_vad_processor.py index 2acb4f29d..afb6e1482 100644 --- a/tests/test_vad_processor.py +++ b/tests/test_vad_processor.py @@ -10,6 +10,7 @@ from typing import List from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState from pipecat.frames.frames import ( InputAudioRawFrame, + SpeechControlParamsFrame, UserSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, @@ -52,7 +53,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase): await run_test( processor, frames_to_send=[self._make_audio_frame()], - expected_down_frames=[InputAudioRawFrame], + expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame], ) async def test_pushes_started_speaking_frame(self): @@ -60,14 +61,16 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase): analyzer = MockVADAnalyzer([VADState.QUIET, VADState.SPEAKING]) processor = VADProcessor(vad_analyzer=analyzer) + # Audio frames are forwarded first, then VAD processes and broadcasts VAD frames await run_test( processor, frames_to_send=[self._make_audio_frame(), self._make_audio_frame()], expected_down_frames=[ + SpeechControlParamsFrame, + InputAudioRawFrame, InputAudioRawFrame, VADUserStartedSpeakingFrame, UserSpeakingFrame, - InputAudioRawFrame, ], ) @@ -76,15 +79,17 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase): analyzer = MockVADAnalyzer([VADState.SPEAKING, VADState.QUIET]) processor = VADProcessor(vad_analyzer=analyzer) + # Audio frames are forwarded first, then VAD processes and broadcasts VAD frames await run_test( processor, frames_to_send=[self._make_audio_frame(), self._make_audio_frame()], expected_down_frames=[ + SpeechControlParamsFrame, + InputAudioRawFrame, VADUserStartedSpeakingFrame, UserSpeakingFrame, InputAudioRawFrame, VADUserStoppedSpeakingFrame, - InputAudioRawFrame, ], ) @@ -93,15 +98,17 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase): analyzer = MockVADAnalyzer([VADState.SPEAKING, VADState.SPEAKING]) processor = VADProcessor(vad_analyzer=analyzer) + # Audio frames are forwarded first, then VAD processes and broadcasts VAD frames await run_test( processor, frames_to_send=[self._make_audio_frame(), self._make_audio_frame()], expected_down_frames=[ + SpeechControlParamsFrame, + InputAudioRawFrame, VADUserStartedSpeakingFrame, UserSpeakingFrame, InputAudioRawFrame, UserSpeakingFrame, - InputAudioRawFrame, ], ) @@ -113,7 +120,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase): await run_test( processor, frames_to_send=[self._make_audio_frame()], - expected_down_frames=[InputAudioRawFrame], + expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame], ) async def test_no_vad_frames_on_stopping_state(self): @@ -124,7 +131,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase): await run_test( processor, frames_to_send=[self._make_audio_frame()], - expected_down_frames=[InputAudioRawFrame], + expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame], ) async def test_no_vad_frames_when_quiet(self): @@ -135,7 +142,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase): await run_test( processor, frames_to_send=[self._make_audio_frame(), self._make_audio_frame()], - expected_down_frames=[InputAudioRawFrame, InputAudioRawFrame], + expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame, InputAudioRawFrame], )