Merge pull request #3628 from pipecat-ai/mb/broadcast-speech-control-params-frame

Fix: Broadcast SpeechControlParamsFrame from VADController

changelog/3628.fixed.md

@@ -0,0 +1 @@
+- Fixed `VADController` not broadcasting `SpeechControlParamsFrame` on startup, which prevented STT services from receiving VAD params needed for TTFB measurement.

src/pipecat/audio/vad/vad_controller.py

@@ -106,6 +106,8 @@ class VADController(BaseObject):
 
     async def _start(self, frame: StartFrame):
         self._vad_analyzer.set_sample_rate(frame.audio_in_sample_rate)
+        # Broadcast initial VAD params so other services (e.g. STT) can use them
+        await self.broadcast_frame(SpeechControlParamsFrame, vad_params=self._vad_analyzer.params)
 
     async def _handle_audio(self, frame: InputAudioRawFrame):
         """Process an audio chunk and emit speech events as needed.

src/pipecat/processors/audio/vad_processor.py

@@ -93,8 +93,10 @@ class VADProcessor(FrameProcessor):
         """
         await super().process_frame(frame, direction)
 
+        # Forward the frame first, then let VAD controller process. This ensures:
+        # 1. StartFrame reaches downstream before SpeechControlParamsFrame is broadcast
+        # 2. Audio flows through immediately while VAD detection happens after
+        await self.push_frame(frame, direction)
+
         # Let the VAD controller handle the frame
         await self._vad_controller.process_frame(frame)
-
-        # Always forward the frame
-        await self.push_frame(frame, direction)

tests/test_vad_controller.py

@@ -7,9 +7,9 @@
 import unittest
 from typing import List
 
-from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
+from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADParams, VADState
 from pipecat.audio.vad.vad_controller import VADController
-from pipecat.frames.frames import Frame, InputAudioRawFrame, StartFrame
+from pipecat.frames.frames import Frame, InputAudioRawFrame, SpeechControlParamsFrame, StartFrame
 from pipecat.processors.frame_processor import FrameDirection
 
 
@@ -185,6 +185,26 @@ class TestVADController(unittest.IsolatedAsyncioTestCase):
         await controller.process_frame(audio_frame)
         self.assertEqual(events_triggered, [])
 
+    async def test_start_frame_broadcasts_vad_params(self):
+        """Test that StartFrame triggers broadcast of SpeechControlParamsFrame with VAD params."""
+        analyzer = MockVADAnalyzer()
+        controller = VADController(analyzer)
+
+        broadcast_calls: List[tuple] = []
+
+        @controller.event_handler("on_broadcast_frame")
+        async def on_broadcast_frame(_controller, frame_cls, **kwargs):
+            broadcast_calls.append((frame_cls, kwargs))
+
+        start_frame = StartFrame(audio_in_sample_rate=16000, audio_out_sample_rate=16000)
+        await controller.process_frame(start_frame)
+
+        # Should have broadcast SpeechControlParamsFrame with VAD params
+        self.assertEqual(len(broadcast_calls), 1)
+        self.assertEqual(broadcast_calls[0][0], SpeechControlParamsFrame)
+        self.assertIn("vad_params", broadcast_calls[0][1])
+        self.assertIsInstance(broadcast_calls[0][1]["vad_params"], VADParams)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_vad_processor.py

@@ -10,6 +10,7 @@ from typing import List
 from pipecat.audio.vad.vad_analyzer import VADAnalyzer, VADState
 from pipecat.frames.frames import (
     InputAudioRawFrame,
+    SpeechControlParamsFrame,
     UserSpeakingFrame,
     VADUserStartedSpeakingFrame,
     VADUserStoppedSpeakingFrame,
@@ -52,7 +53,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase):
         await run_test(
             processor,
             frames_to_send=[self._make_audio_frame()],
-            expected_down_frames=[InputAudioRawFrame],
+            expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame],
         )
 
     async def test_pushes_started_speaking_frame(self):
@@ -60,14 +61,16 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase):
         analyzer = MockVADAnalyzer([VADState.QUIET, VADState.SPEAKING])
         processor = VADProcessor(vad_analyzer=analyzer)
 
+        # Audio frames are forwarded first, then VAD processes and broadcasts VAD frames
         await run_test(
             processor,
             frames_to_send=[self._make_audio_frame(), self._make_audio_frame()],
             expected_down_frames=[
+                SpeechControlParamsFrame,
+                InputAudioRawFrame,
                 InputAudioRawFrame,
                 VADUserStartedSpeakingFrame,
                 UserSpeakingFrame,
-                InputAudioRawFrame,
             ],
         )
 
@@ -76,15 +79,17 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase):
         analyzer = MockVADAnalyzer([VADState.SPEAKING, VADState.QUIET])
         processor = VADProcessor(vad_analyzer=analyzer)
 
+        # Audio frames are forwarded first, then VAD processes and broadcasts VAD frames
         await run_test(
             processor,
             frames_to_send=[self._make_audio_frame(), self._make_audio_frame()],
             expected_down_frames=[
+                SpeechControlParamsFrame,
+                InputAudioRawFrame,
                 VADUserStartedSpeakingFrame,
                 UserSpeakingFrame,
                 InputAudioRawFrame,
                 VADUserStoppedSpeakingFrame,
-                InputAudioRawFrame,
             ],
         )
 
@@ -93,15 +98,17 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase):
         analyzer = MockVADAnalyzer([VADState.SPEAKING, VADState.SPEAKING])
         processor = VADProcessor(vad_analyzer=analyzer)
 
+        # Audio frames are forwarded first, then VAD processes and broadcasts VAD frames
         await run_test(
             processor,
             frames_to_send=[self._make_audio_frame(), self._make_audio_frame()],
             expected_down_frames=[
+                SpeechControlParamsFrame,
+                InputAudioRawFrame,
                 VADUserStartedSpeakingFrame,
                 UserSpeakingFrame,
                 InputAudioRawFrame,
                 UserSpeakingFrame,
-                InputAudioRawFrame,
             ],
         )
 
@@ -113,7 +120,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase):
         await run_test(
             processor,
             frames_to_send=[self._make_audio_frame()],
-            expected_down_frames=[InputAudioRawFrame],
+            expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame],
         )
 
     async def test_no_vad_frames_on_stopping_state(self):
@@ -124,7 +131,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase):
         await run_test(
             processor,
             frames_to_send=[self._make_audio_frame()],
-            expected_down_frames=[InputAudioRawFrame],
+            expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame],
         )
 
     async def test_no_vad_frames_when_quiet(self):
@@ -135,7 +142,7 @@ class TestVADProcessor(unittest.IsolatedAsyncioTestCase):
         await run_test(
             processor,
             frames_to_send=[self._make_audio_frame(), self._make_audio_frame()],
-            expected_down_frames=[InputAudioRawFrame, InputAudioRawFrame],
+            expected_down_frames=[SpeechControlParamsFrame, InputAudioRawFrame, InputAudioRawFrame],
         )