Triggering to check if the turn is complete each time the user stops speaking based on the vad

examples/foundational/38a-local-smart-turn.py

@@ -11,6 +11,7 @@ from loguru import logger
 
 from pipecat.audio.turn.local_smart_turn import LocalSmartTurnAnalyzer
 from pipecat.audio.vad.silero import SileroVADAnalyzer
+from pipecat.audio.vad.vad_analyzer import VADParams
 from pipecat.pipeline.pipeline import Pipeline
 from pipecat.pipeline.runner import PipelineRunner
 from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -34,7 +35,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection):
             audio_in_enabled=True,
             audio_out_enabled=True,
             vad_enabled=True,
-            vad_analyzer=SileroVADAnalyzer(),
+            vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
             vad_audio_passthrough=True,
             end_of_turn_analyzer=LocalSmartTurnAnalyzer(),
         ),

src/pipecat/audio/turn/base_turn_analyzer.py

@@ -36,5 +36,9 @@ class BaseEndOfTurnAnalyzer(ABC):
         self._chunk_size_ms = chunk_size_ms
 
     @abstractmethod
-    def analyze_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
+    def append_audio(self, buffer: bytes, is_speech: bool):
+        pass
+
+    @abstractmethod
+    def analyze_end_of_turn(self) -> EndOfTurnState:
         pass

src/pipecat/audio/turn/local_smart_turn.py

@@ -28,7 +28,7 @@ except ModuleNotFoundError as e:
 # TODO: we should convert all this to params
 STOP_MS = 1000
 PRE_SPEECH_MS = 200
-MAX_DURATION_SECONDS = 16  # Maximum duration for the smart turn model
+MAX_DURATION_SECONDS = 8  # Maximum duration for the smart turn model
 
 
 class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
@@ -68,11 +68,8 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
         self._silence_frames = 0
         self._speech_start_time = None
 
-    def analyze_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
+    def append_audio(self, buffer: bytes, is_speech: bool):
-        state = EndOfTurnState.INCOMPLETE
-
         audio_int16 = np.frombuffer(buffer, dtype=np.int16)
-
         # Divide by 32768 because we have signed 16-bit data.
         audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
 
@@ -87,18 +84,6 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
             if self._speech_triggered:
                 self._audio_buffer.append((time.time(), audio_float32))
                 self._silence_frames += 1
-                if self._silence_frames * self._chunk_size_ms >= STOP_MS:
-                    self._speech_triggered = False
-
-                    # TODO: do we need to stop or do something to prevent ??
-
-                    state = self._process_speech_segment(
-                        self._audio_buffer, self._speech_start_time
-                    )
-                    self._audio_buffer = []
-                    self._speech_start_time = None
-
-                    # TODO: same here for restart
             else:
                 # Keep buffering some silence before potential speech starts
                 self._audio_buffer.append((time.time(), audio_float32))
@@ -111,16 +96,30 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
                 ):
                     self._audio_buffer.pop(0)
 
+    def analyze_end_of_turn(self) -> EndOfTurnState:
+        logger.debug("Analyzing End of Turn...")
+        if self._silence_frames * self._chunk_size_ms >= STOP_MS:
+            logger.debug("End of Turn complete due to STOP_MS.")
+            state = EndOfTurnState.COMPLETE
+        else:
+            state = self._process_speech_segment(self._audio_buffer)
+
+        if state == EndOfTurnState.COMPLETE:
+            self._speech_triggered = False
+            self._audio_buffer = []
+            self._speech_start_time = None
+
+        logger.debug(f"End of Turn result: {state}")
         return state
 
-    def _process_speech_segment(self, audio_buffer, speech_start_time) -> EndOfTurnState:
+    def _process_speech_segment(self, audio_buffer) -> EndOfTurnState:
         state = EndOfTurnState.INCOMPLETE
 
         if not audio_buffer:
             return state
 
         # Find start and end indices for the segment
-        start_time = speech_start_time - (PRE_SPEECH_MS / 1000)
+        start_time = self._speech_start_time - (PRE_SPEECH_MS / 1000)
         start_index = 0
         for i, (t, _) in enumerate(audio_buffer):
             if t >= start_time:

src/pipecat/transports/base_input.py

@@ -217,27 +217,16 @@ class BaseInputTransport(FrameProcessor):
             vad_state = new_vad_state
         return vad_state
 
-    async def _end_of_turn_analyze(
+    async def _handle_end_of_turn(self, end_of_turn_state: EndOfTurnState):
-        self, audio_frame: InputAudioRawFrame, is_speech: bool
+        state = end_of_turn_state
-    ) -> EndOfTurnState:
-        state = EndOfTurnState.INCOMPLETE
         if self.end_of_turn_analyzer:
-            state = await self.get_event_loop().run_in_executor(
+            new_state = await self.get_event_loop().run_in_executor(
-                self._executor,
+                self._executor, self.end_of_turn_analyzer.analyze_end_of_turn
-                self.end_of_turn_analyzer.analyze_audio,
-                audio_frame.audio,
-                is_speech,
             )
+            if new_state != state and new_state == EndOfTurnState.COMPLETE:
+                await self._handle_user_interruption(UserEndOfTurnFrame())
         return state
 
-    async def _handle_end_of_turn(
-        self, audio_frame: InputAudioRawFrame, end_of_turn_state: EndOfTurnState, is_speech: bool
-    ):
-        new_eot_state = await self._end_of_turn_analyze(audio_frame, is_speech)
-        if new_eot_state != end_of_turn_state:
-            await self._handle_user_interruption(UserEndOfTurnFrame())
-        return new_eot_state
-
     async def _audio_task_handler(self):
         vad_state: VADState = VADState.QUIET
         end_of_turn_state: EndOfTurnState = EndOfTurnState.INCOMPLETE
@@ -252,15 +241,16 @@ class BaseInputTransport(FrameProcessor):
 
             # Check VAD and push event if necessary. We just care about
             # changes from QUIET to SPEAKING and vice versa.
+            previous_vad_state = vad_state
             if self._params.vad_enabled:
                 vad_state = await self._handle_vad(frame, vad_state)
                 audio_passthrough = self._params.vad_audio_passthrough
 
             if self._params.end_of_turn_analyzer:
                 is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
-                end_of_turn_state = await self._handle_end_of_turn(
+                self._params.end_of_turn_analyzer.append_audio(frame.audio, is_speech)
-                    frame, end_of_turn_state, is_speech
+                if vad_state == VADState.QUIET and vad_state != previous_vad_state:
-                )
+                    end_of_turn_state = await self._handle_end_of_turn(end_of_turn_state)
 
             # Push audio downstream if passthrough.
             if audio_passthrough: