wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

MinWordsUserTurnStartStrategy: single word interrupt if bot not speaking

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2025-12-22 14:32:21 -08:00
parent 08a9b434c1
commit a0d5ee3873
3 changed files with 80 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/pipecat/audio/turn/smart_turn/base_smart_turn.py
View File

@@ -19,7 +19,6 @@ from typing import Any, Dict, Optional, Tuple
import numpy as np
from loguru import logger
from pydantic import BaseModel
from pipecat.audio.turn.base_turn_analyzer import BaseTurnAnalyzer, BaseTurnParams, EndOfTurnState
from pipecat.metrics.metrics import MetricsData, SmartTurnMetricsData

50
src/pipecat/turns/user/min_words_user_turn_start_strategy.py
View File

@@ -8,7 +8,13 @@
from loguru import logger
from pipecat.frames.frames import Frame, InterimTranscriptionFrame, TranscriptionFrame
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
Frame,
InterimTranscriptionFrame,
TranscriptionFrame,
)
from pipecat.turns.user.base_user_turn_start_strategy import BaseUserTurnStartStrategy
@@ -33,12 +39,14 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy):
super().__init__()
self._min_words = min_words
self._use_interim = use_interim
self._bot_speaking = False
self._text = ""
async def reset(self):
"""Reset the strategy to its initial state."""
await super().reset()
self._text = ""
self._bot_speaking = False
async def process_frame(self, frame: Frame):
"""Process an incoming frame to detect the start of a user turn.
@@ -51,11 +59,35 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy):
"""
await super().process_frame(frame)
if isinstance(frame, TranscriptionFrame):
if isinstance(frame, BotStartedSpeakingFrame):
await self._handle_bot_started_speaking(frame)
elif isinstance(frame, BotStoppedSpeakingFrame):
await self._handle_bot_stopped_speaking(frame)
elif isinstance(frame, TranscriptionFrame):
await self._handle_transcription(frame)
elif isinstance(frame, InterimTranscriptionFrame) and self._use_interim:
await self._handle_interim_transcription(frame)
async def _handle_bot_started_speaking(self, frame: BotStartedSpeakingFrame):
"""Handle bot started speaking frame.
If the bot is speaking we want to interrupt using min words.
Args:
frame: The frame to be processed.
"""
self._bot_speaking = True
async def _handle_bot_stopped_speaking(self, frame: BotStoppedSpeakingFrame):
"""Handle bot started speaking frame.
If the bot is not speaking we want to interrupt if we get a single word.
Args:
frame: The frame to be processed.
"""
self._bot_speaking = False
async def _handle_transcription(self, frame: TranscriptionFrame):
"""Handle a completed transcription frame and check word count.
@@ -64,11 +96,14 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy):
"""
self._text += frame.text
min_words = self._min_words if self._bot_speaking else 1
word_count = len(self._text.split())
should_trigger = word_count >= self._min_words
should_trigger = word_count >= min_words
logger.debug(
f"{self} should_trigger={should_trigger} num_spoken_words={word_count} min_words={self._min_words}"
f"{self} should_trigger={should_trigger} num_spoken_words={word_count} "
f"min_words={min_words} bot_speaking={self._bot_speaking}"
)
if should_trigger:
@@ -80,11 +115,14 @@ class MinWordsUserTurnStartStrategy(BaseUserTurnStartStrategy):
Args:
frame: The interim transcription frame to be processed.
"""
min_words = self._min_words if self._bot_speaking else 1
word_count = len(frame.text.split())
should_trigger = word_count >= self._min_words
should_trigger = word_count >= min_words
logger.debug(
f"{self} interim=True should_trigger={should_trigger} num_spoken_words={word_count} min_words={self._min_words}"
f"{self} interim=True should_trigger={should_trigger} num_spoken_words={word_count} "
f"min_words={min_words} bot_speaking={self._bot_speaking}"
)
if should_trigger:

39
tests/test_user_turn_start_strategy.py
View File

@@ -21,7 +21,7 @@ from pipecat.turns.user.vad_user_turn_start_strategy import VADUserTurnStartStra
class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase):
async def test_only_transcriptions(self):
async def test_bot_speaking_transcriptions(self):
strategy = MinWordsUserTurnStartStrategy(min_words=2)
should_start = None
@@ -31,6 +31,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase):
nonlocal should_start
should_start = True
await strategy.process_frame(BotStartedSpeakingFrame())
await strategy.process_frame(TranscriptionFrame(text="Hello", user_id="cat", timestamp=""))
self.assertFalse(should_start)
@@ -43,6 +44,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase):
should_start = None
await strategy.reset()
await strategy.process_frame(BotStartedSpeakingFrame())
await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp=""))
self.assertFalse(should_start)
@@ -51,7 +53,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase):
)
self.assertTrue(should_start)
async def test_only_interim_transcriptions(self):
async def test_bot_speaking_interim_transcriptions(self):
strategy = MinWordsUserTurnStartStrategy(min_words=2)
should_start = None
@@ -61,17 +63,19 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase):
nonlocal should_start
should_start = True
await strategy.process_frame(BotStartedSpeakingFrame())
await strategy.process_frame(
InterimTranscriptionFrame(text="Hello", user_id="cat", timestamp="")
)
self.assertFalse(should_start)
await strategy.process_frame(BotStartedSpeakingFrame())
await strategy.process_frame(
InterimTranscriptionFrame(text="Hello there!", user_id="cat", timestamp="")
)
self.assertTrue(should_start)
async def test_all_transcriptions(self):
async def test_bot_speaking_all_transcriptions(self):
strategy = MinWordsUserTurnStartStrategy(min_words=2)
should_start = None
@@ -81,6 +85,7 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase):
nonlocal should_start
should_start = True
await strategy.process_frame(BotStartedSpeakingFrame())
await strategy.process_frame(
InterimTranscriptionFrame(text="Hello", user_id="cat", timestamp="")
)
@@ -91,6 +96,34 @@ class TestMinWordsInterruptionStrategy(unittest.IsolatedAsyncioTestCase):
)
self.assertTrue(should_start)
async def test_bot_not_speaking_transcriptions(self):
strategy = MinWordsUserTurnStartStrategy(min_words=2)
should_start = None
@strategy.event_handler("on_user_turn_started")
async def on_user_turn_started(strategy):
nonlocal should_start
should_start = True
await strategy.process_frame(TranscriptionFrame(text="Hello", user_id="cat", timestamp=""))
self.assertTrue(should_start)
async def test_bot_not_speaking_interim_transcriptions(self):
strategy = MinWordsUserTurnStartStrategy(min_words=2)
should_start = None
@strategy.event_handler("on_user_turn_started")
async def on_user_turn_started(strategy):
nonlocal should_start
should_start = True
await strategy.process_frame(
InterimTranscriptionFrame(text="Hello", user_id="cat", timestamp="")
)
self.assertTrue(should_start)
class TestVADUserTurnStartStrategy(unittest.IsolatedAsyncioTestCase):
async def test_vad_strategy(self):