wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Triggering to check if the turn is complete each time the user stops speaking based on the vad

Browse Source
This commit is contained in:
Filipi Fuchter
2025-04-15 17:28:00 -03:00
parent e6325a8229
commit 11b6de0900
4 changed files with 35 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
examples/foundational/38a-local-smart-turn.py
View File

@@ -11,6 +11,7 @@ from loguru import logger
from pipecat.audio.turn.local_smart_turn import LocalSmartTurnAnalyzer
from pipecat.audio.vad.silero import SileroVADAnalyzer
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.pipeline.pipeline import Pipeline
from pipecat.pipeline.runner import PipelineRunner
from pipecat.pipeline.task import PipelineParams, PipelineTask
@@ -34,7 +35,7 @@ async def run_bot(webrtc_connection: SmallWebRTCConnection):
audio_in_enabled=True,
audio_out_enabled=True,
vad_enabled=True,
vad_analyzer=SileroVADAnalyzer(),
vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.2)),
vad_audio_passthrough=True,
end_of_turn_analyzer=LocalSmartTurnAnalyzer(),
),

6
src/pipecat/audio/turn/base_turn_analyzer.py
View File

@@ -36,5 +36,9 @@ class BaseEndOfTurnAnalyzer(ABC):
self._chunk_size_ms = chunk_size_ms
@abstractmethod
def analyze_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
def append_audio(self, buffer: bytes, is_speech: bool):
pass
@abstractmethod
def analyze_end_of_turn(self) -> EndOfTurnState:
pass

37
src/pipecat/audio/turn/local_smart_turn.py
View File

@@ -28,7 +28,7 @@ except ModuleNotFoundError as e:
# TODO: we should convert all this to params
STOP_MS = 1000
PRE_SPEECH_MS = 200
MAX_DURATION_SECONDS = 16 # Maximum duration for the smart turn model
MAX_DURATION_SECONDS = 8 # Maximum duration for the smart turn model
class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
@@ -68,11 +68,8 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
self._silence_frames = 0
self._speech_start_time = None
def analyze_audio(self, buffer: bytes, is_speech: bool) -> EndOfTurnState:
state = EndOfTurnState.INCOMPLETE
def append_audio(self, buffer: bytes, is_speech: bool):
audio_int16 = np.frombuffer(buffer, dtype=np.int16)
# Divide by 32768 because we have signed 16-bit data.
audio_float32 = np.frombuffer(audio_int16, dtype=np.int16).astype(np.float32) / 32768.0
@@ -87,18 +84,6 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
if self._speech_triggered:
self._audio_buffer.append((time.time(), audio_float32))
self._silence_frames += 1
if self._silence_frames * self._chunk_size_ms >= STOP_MS:
self._speech_triggered = False
# TODO: do we need to stop or do something to prevent ??
state = self._process_speech_segment(
self._audio_buffer, self._speech_start_time
)
self._audio_buffer = []
self._speech_start_time = None
# TODO: same here for restart
else:
# Keep buffering some silence before potential speech starts
self._audio_buffer.append((time.time(), audio_float32))
@@ -111,16 +96,30 @@ class LocalSmartTurnAnalyzer(BaseEndOfTurnAnalyzer):
):
self._audio_buffer.pop(0)
def analyze_end_of_turn(self) -> EndOfTurnState:
logger.debug("Analyzing End of Turn...")
if self._silence_frames * self._chunk_size_ms >= STOP_MS:
logger.debug("End of Turn complete due to STOP_MS.")
state = EndOfTurnState.COMPLETE
else:
state = self._process_speech_segment(self._audio_buffer)
if state == EndOfTurnState.COMPLETE:
self._speech_triggered = False
self._audio_buffer = []
self._speech_start_time = None
logger.debug(f"End of Turn result: {state}")
return state
def _process_speech_segment(self, audio_buffer, speech_start_time) -> EndOfTurnState:
def _process_speech_segment(self, audio_buffer) -> EndOfTurnState:
state = EndOfTurnState.INCOMPLETE
if not audio_buffer:
return state
# Find start and end indices for the segment
start_time = speech_start_time - (PRE_SPEECH_MS / 1000)
start_time = self._speech_start_time - (PRE_SPEECH_MS / 1000)
start_index = 0
for i, (t, _) in enumerate(audio_buffer):
if t >= start_time:

30
src/pipecat/transports/base_input.py
View File

@@ -217,27 +217,16 @@ class BaseInputTransport(FrameProcessor):
vad_state = new_vad_state
return vad_state
async def _end_of_turn_analyze(
self, audio_frame: InputAudioRawFrame, is_speech: bool
) -> EndOfTurnState:
state = EndOfTurnState.INCOMPLETE
async def _handle_end_of_turn(self, end_of_turn_state: EndOfTurnState):
state = end_of_turn_state
if self.end_of_turn_analyzer:
state = await self.get_event_loop().run_in_executor(
self._executor,
self.end_of_turn_analyzer.analyze_audio,
audio_frame.audio,
is_speech,
new_state = await self.get_event_loop().run_in_executor(
self._executor, self.end_of_turn_analyzer.analyze_end_of_turn
)
if new_state != state and new_state == EndOfTurnState.COMPLETE:
await self._handle_user_interruption(UserEndOfTurnFrame())
return state
async def _handle_end_of_turn(
self, audio_frame: InputAudioRawFrame, end_of_turn_state: EndOfTurnState, is_speech: bool
):
new_eot_state = await self._end_of_turn_analyze(audio_frame, is_speech)
if new_eot_state != end_of_turn_state:
await self._handle_user_interruption(UserEndOfTurnFrame())
return new_eot_state
async def _audio_task_handler(self):
vad_state: VADState = VADState.QUIET
end_of_turn_state: EndOfTurnState = EndOfTurnState.INCOMPLETE
@@ -252,15 +241,16 @@ class BaseInputTransport(FrameProcessor):
# Check VAD and push event if necessary. We just care about
# changes from QUIET to SPEAKING and vice versa.
previous_vad_state = vad_state
if self._params.vad_enabled:
vad_state = await self._handle_vad(frame, vad_state)
audio_passthrough = self._params.vad_audio_passthrough
if self._params.end_of_turn_analyzer:
is_speech = vad_state == VADState.SPEAKING or vad_state == VADState.STARTING
end_of_turn_state = await self._handle_end_of_turn(
frame, end_of_turn_state, is_speech
)
self._params.end_of_turn_analyzer.append_audio(frame.audio, is_speech)
if vad_state == VADState.QUIET and vad_state != previous_vad_state:
end_of_turn_state = await self._handle_end_of_turn(end_of_turn_state)
# Push audio downstream if passthrough.
if audio_passthrough: