diff --git a/src/pipecat/audio/turn/smart_turn/base_smart_turn.py b/src/pipecat/audio/turn/smart_turn/base_smart_turn.py index 5cb832430..9442f66bc 100644 --- a/src/pipecat/audio/turn/smart_turn/base_smart_turn.py +++ b/src/pipecat/audio/turn/smart_turn/base_smart_turn.py @@ -19,7 +19,6 @@ from typing import Any, Dict, Optional, Tuple import numpy as np from loguru import logger -from pydantic import BaseModel from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData diff --git a/src/pipecat/turns/user/min_words_user_turn_start_strategy.py b/src/pipecat/turns/user/min_words_user_turn_start_strategy.py index 61204aec8..64b01ae66 100644 --- a/src/pipecat/turns/user/min_words_user_turn_start_strategy.py +++ b/src/pipecat/turns/user/min_words_user_turn_start_strategy.py @@ -8,7 +8,13 @@ from loguru import logger -from pipecat.frames.frames import Frame, InterimTranscriptionFrame, TranscriptionFrame +from pipecat.frames.frames import ( + BotStartedSpeakingFrame, + BotStoppedSpeakingFrame, + Frame, + InterimTranscriptionFrame, + TranscriptionFrame, +) from pipecat.turns.user.base_user_turn_start_strategy import BaseUserTurnStartStrategy @@ -33,12 +39,14 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy): super().__init__() self._min_words = min_words self._use_interim = use_interim + self._bot_speaking = False self._text = "" async def reset(self): """Reset the strategy to its initial state.""" await super().reset() self._text = "" + self._bot_speaking = False async def process_frame(self, frame: Frame): """Process an incoming frame to detect the start of a user turn. @@ -51,11 +59,35 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy): """ await super().process_frame(frame) - if isinstance(frame, TranscriptionFrame): + if isinstance(frame, BotStartedSpeakingFrame): + await self._handle_bot_started_speaking(frame) + elif isinstance(frame, BotStoppedSpeakingFrame): + await self._handle_bot_stopped_speaking(frame) + elif isinstance(frame, TranscriptionFrame): await self._handle_transcription(frame) elif isinstance(frame, InterimTranscriptionFrame) and self._use_interim: await self._handle_interim_transcription(frame) + async def _handle_bot_started_speaking(self, frame: BotStartedSpeakingFrame): + """Handle bot started speaking frame. + + If the bot is speaking we want to interrupt using min words. + + Args: + frame: The frame to be processed. + """ + self._bot_speaking = True + + async def _handle_bot_stopped_speaking(self, frame: BotStoppedSpeakingFrame): + """Handle bot started speaking frame. + + If the bot is not speaking we want to interrupt if we get a single word. + + Args: + frame: The frame to be processed. + """ + self._bot_speaking = False + async def _handle_transcription(self, frame: TranscriptionFrame): """Handle a completed transcription frame and check word count. @@ -64,11 +96,14 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy): """ self._text += frame.text + min_words = self._min_words if self._bot_speaking else 1 + word_count = len(self._text.split()) - should_trigger = word_count >= self._min_words + should_trigger = word_count >= min_words logger.debug( - f"{self} should_trigger={should_trigger} num_spoken_words={word_count} min_words={self._min_words}" + f"{self} should_trigger={should_trigger} num_spoken_words={word_count} " + f"min_words={min_words} bot_speaking={self._bot_speaking}" ) if should_trigger: @@ -80,11 +115,14 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy): Args: frame: The interim transcription frame to be processed. """ + min_words = self._min_words if self._bot_speaking else 1 + word_count = len(frame.text.split()) - should_trigger = word_count >= self._min_words + should_trigger = word_count >= min_words logger.debug( - f"{self} interim=True should_trigger={should_trigger} num_spoken_words={word_count} min_words={self._min_words}" + f"{self} interim=True should_trigger={should_trigger} num_spoken_words={word_count} " + f"min_words={min_words} bot_speaking={self._bot_speaking}" ) if should_trigger: diff --git a/tests/test_user_turn_start_strategy.py b/tests/test_user_turn_start_strategy.py index 9402c1793..94df71363 100644 --- a/tests/test_user_turn_start_strategy.py +++ b/tests/test_user_turn_start_strategy.py @@ -21,7 +21,7 @@ from pipecat.turns.user.vad_user_turn_start_strategy import VADUserTurnStartStra class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase): - async def test_only_transcriptions(self): + async def test_bot_speaking_transcriptions(self): strategy = MinWordsUserTurnStartStrategy(min_words=2) should_start = None @@ -31,6 +31,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase): nonlocal should_start should_start = True + await strategy.process_frame(BotStartedSpeakingFrame()) await strategy.process_frame(TranscriptionFrame(text="Hello", user_id="cat", timestamp="")) self.assertFalse(should_start) @@ -43,6 +44,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase): should_start = None await strategy.reset() + await strategy.process_frame(BotStartedSpeakingFrame()) await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp="")) self.assertFalse(should_start) @@ -51,7 +53,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase): ) self.assertTrue(should_start) - async def test_only_interim_transcriptions(self): + async def test_bot_speaking_interim_transcriptions(self): strategy = MinWordsUserTurnStartStrategy(min_words=2) should_start = None @@ -61,17 +63,19 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase): nonlocal should_start should_start = True + await strategy.process_frame(BotStartedSpeakingFrame()) await strategy.process_frame( InterimTranscriptionFrame(text="Hello", user_id="cat", timestamp="") ) self.assertFalse(should_start) + await strategy.process_frame(BotStartedSpeakingFrame()) await strategy.process_frame( InterimTranscriptionFrame(text="Hello there!", user_id="cat", timestamp="") ) self.assertTrue(should_start) - async def test_all_transcriptions(self): + async def test_bot_speaking_all_transcriptions(self): strategy = MinWordsUserTurnStartStrategy(min_words=2) should_start = None @@ -81,6 +85,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase): nonlocal should_start should_start = True + await strategy.process_frame(BotStartedSpeakingFrame()) await strategy.process_frame( InterimTranscriptionFrame(text="Hello", user_id="cat", timestamp="") ) @@ -91,6 +96,34 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase): ) self.assertTrue(should_start) + async def test_bot_not_speaking_transcriptions(self): + strategy = MinWordsUserTurnStartStrategy(min_words=2) + + should_start = None + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy): + nonlocal should_start + should_start = True + + await strategy.process_frame(TranscriptionFrame(text="Hello", user_id="cat", timestamp="")) + self.assertTrue(should_start) + + async def test_bot_not_speaking_interim_transcriptions(self): + strategy = MinWordsUserTurnStartStrategy(min_words=2) + + should_start = None + + @strategy.event_handler("on_user_turn_started") + async def on_user_turn_started(strategy): + nonlocal should_start + should_start = True + + await strategy.process_frame( + InterimTranscriptionFrame(text="Hello", user_id="cat", timestamp="") + ) + self.assertTrue(should_start) + class TestVADUserTurnStartStrategy(unittest.IsolatedAsyncioTestCase): async def test_vad_strategy(self):