diff --git a/examples/foundational/38a-local-smart-turn.py b/examples/foundational/38a-local-smart-turn.py index a9bf8e293..490a26684 100644 --- a/examples/foundational/38a-local-smart-turn.py +++ b/examples/foundational/38a-local-smart-turn.py @@ -11,6 +11,7 @@ from loguru import logger from pipecat.audio.turn.local_smart_turn import LocalSmartTurnAnalyzer from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.audio.vad.vad_analyzer import VADParams from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.runner import PipelineRunner from pipecat.pipeline.task import PipelineParams, PipelineTask @@ -34,7 +35,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection): audio_in_enabled=True, audio_out_enabled=True, vad_enabled=True, - vad_analyzer=SileroVADAnalyzer(), + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)), vad_audio_passthrough=True, end_of_turn_analyzer=LocalSmartTurnAnalyzer(), ), diff --git a/src/pipecat/audio/turn/base_turn_analyzer.py b/src/pipecat/audio/turn/base_turn_analyzer.py index eeb9bfeb8..1e6a964f9 100644 --- a/src/pipecat/audio/turn/base_turn_analyzer.py +++ b/src/pipecat/audio/turn/base_turn_analyzer.py @@ -36,5 +36,9 @@ class BaseEndOfTurnAnalyzer(ABC): self._chunk_size_ms = chunk_size_ms @abstractmethod - def analyze_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState: + def append_audio(self, buffer: bytes, is_speech: bool): + pass + + @abstractmethod + def analyze_end_of_turn(self) -> EndOfTurnState: pass diff --git a/src/pipecat/audio/turn/local_smart_turn.py b/src/pipecat/audio/turn/local_smart_turn.py index d7733adc3..b831fffeb 100644 --- a/src/pipecat/audio/turn/local_smart_turn.py +++ b/src/pipecat/audio/turn/local_smart_turn.py @@ -28,7 +28,7 @@ except ModuleNotFoundError as e: # TODO: we should convert all this to params STOP_MS = 1000 PRE_SPEECH_MS = 200 -MAX_DURATION_SECONDS = 16 # Maximum duration for the smart turn model +MAX_DURATION_SECONDS = 8 # Maximum duration for the smart turn model class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer): @@ -68,11 +68,8 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer): self._silence_frames = 0 self._speech_start_time = None - def analyze_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState: - state = EndOfTurnState.INCOMPLETE - + def append_audio(self, buffer: bytes, is_speech: bool): audio_int16 = np.frombuffer(buffer, dtype=np.int16) - # Divide by 32768 because we have signed 16-bit data. audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0 @@ -87,18 +84,6 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer): if self._speech_triggered: self._audio_buffer.append((time.time(), audio_float32)) self._silence_frames += 1 - if self._silence_frames * self._chunk_size_ms >= STOP_MS: - self._speech_triggered = False - - # TODO: do we need to stop or do something to prevent ?? - - state = self._process_speech_segment( - self._audio_buffer, self._speech_start_time - ) - self._audio_buffer = [] - self._speech_start_time = None - - # TODO: same here for restart else: # Keep buffering some silence before potential speech starts self._audio_buffer.append((time.time(), audio_float32)) @@ -111,16 +96,30 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer): ): self._audio_buffer.pop(0) + def analyze_end_of_turn(self) -> EndOfTurnState: + logger.debug("Analyzing End of Turn...") + if self._silence_frames * self._chunk_size_ms >= STOP_MS: + logger.debug("End of Turn complete due to STOP_MS.") + state = EndOfTurnState.COMPLETE + else: + state = self._process_speech_segment(self._audio_buffer) + + if state == EndOfTurnState.COMPLETE: + self._speech_triggered = False + self._audio_buffer = [] + self._speech_start_time = None + + logger.debug(f"End of Turn result: {state}") return state - def _process_speech_segment(self, audio_buffer, speech_start_time) -> EndOfTurnState: + def _process_speech_segment(self, audio_buffer) -> EndOfTurnState: state = EndOfTurnState.INCOMPLETE if not audio_buffer: return state # Find start and end indices for the segment - start_time = speech_start_time - (PRE_SPEECH_MS / 1000) + start_time = self._speech_start_time - (PRE_SPEECH_MS / 1000) start_index = 0 for i, (t, _) in enumerate(audio_buffer): if t >= start_time: diff --git a/src/pipecat/transports/base_input.py b/src/pipecat/transports/base_input.py index 4221b2f65..296d0a453 100644 --- a/src/pipecat/transports/base_input.py +++ b/src/pipecat/transports/base_input.py @@ -217,27 +217,16 @@ class BaseInputTransport(FrameProcessor): vad_state = new_vad_state return vad_state - async def _end_of_turn_analyze( - self, audio_frame: InputAudioRawFrame, is_speech: bool - ) -> EndOfTurnState: - state = EndOfTurnState.INCOMPLETE + async def _handle_end_of_turn(self, end_of_turn_state: EndOfTurnState): + state = end_of_turn_state if self.end_of_turn_analyzer: - state = await self.get_event_loop().run_in_executor( - self._executor, - self.end_of_turn_analyzer.analyze_audio, - audio_frame.audio, - is_speech, + new_state = await self.get_event_loop().run_in_executor( + self._executor, self.end_of_turn_analyzer.analyze_end_of_turn ) + if new_state != state and new_state == EndOfTurnState.COMPLETE: + await self._handle_user_interruption(UserEndOfTurnFrame()) return state - async def _handle_end_of_turn( - self, audio_frame: InputAudioRawFrame, end_of_turn_state: EndOfTurnState, is_speech: bool - ): - new_eot_state = await self._end_of_turn_analyze(audio_frame, is_speech) - if new_eot_state != end_of_turn_state: - await self._handle_user_interruption(UserEndOfTurnFrame()) - return new_eot_state - async def _audio_task_handler(self): vad_state: VADState = VADState.QUIET end_of_turn_state: EndOfTurnState = EndOfTurnState.INCOMPLETE @@ -252,15 +241,16 @@ class BaseInputTransport(FrameProcessor): # Check VAD and push event if necessary. We just care about # changes from QUIET to SPEAKING and vice versa. + previous_vad_state = vad_state if self._params.vad_enabled: vad_state = await self._handle_vad(frame, vad_state) audio_passthrough = self._params.vad_audio_passthrough if self._params.end_of_turn_analyzer: is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING - end_of_turn_state = await self._handle_end_of_turn( - frame, end_of_turn_state, is_speech - ) + self._params.end_of_turn_analyzer.append_audio(frame.audio, is_speech) + if vad_state == VADState.QUIET and vad_state != previous_vad_state: + end_of_turn_state = await self._handle_end_of_turn(end_of_turn_state) # Push audio downstream if passthrough. if audio_passthrough: