wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve user turn stop timing by triggering timeout from VAD stop

Browse Source 
Refactor TranscriptionUserTurnStopStrategy and TurnAnalyzerUserTurnStopStrategy
to use VADUserStoppedSpeakingFrame as the ground truth for when speech ended,
rather than triggering timeouts from transcription frames.
This commit is contained in:
Mark Backman
2026-02-03 22:32:34 -05:00
parent 05e2a013b3
commit 34b068d657
47 changed files with 933 additions and 292 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tests/test_context_aggregators_universal.py
View File

@@ -41,7 +41,7 @@ from pipecat.processors.aggregators.llm_response_universal import (
)
from pipecat.tests.utils import SleepFrame, run_test
from pipecat.turns.user_mute import FirstSpeechUserMuteStrategy, FunctionCallUserMuteStrategy
from pipecat.turns.user_stop import TranscriptionUserTurnStopStrategy
from pipecat.turns.user_stop import SpeechTimeoutUserTurnStopStrategy
from pipecat.turns.user_turn_strategies import UserTurnStrategies
USER_TURN_STOP_TIMEOUT = 0.2
@@ -149,7 +149,16 @@ class TestLLMUserAggregator(unittest.IsolatedAsyncioTestCase):
async def test_default_user_turn_strategies(self):
context = LLMContext()
user_aggregator = LLMUserAggregator(context)
user_aggregator = LLMUserAggregator(
context,
params=LLMUserAggregatorParams(
user_turn_strategies=UserTurnStrategies(
stop=[
SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=TRANSCRIPTION_TIMEOUT)
],
),
),
)
should_start = None
should_stop = None
@@ -173,6 +182,8 @@ class TestLLMUserAggregator(unittest.IsolatedAsyncioTestCase):
TranscriptionFrame(text="Hello!", user_id="", timestamp="now"),
SleepFrame(),
VADUserStoppedSpeakingFrame(),
# Wait for user_speech_timeout to elapse
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT + 0.1),
]
expected_down_frames = [
VADUserStartedSpeakingFrame,
@@ -241,7 +252,9 @@ class TestLLMUserAggregator(unittest.IsolatedAsyncioTestCase):
context,
params=LLMUserAggregatorParams(
user_turn_strategies=UserTurnStrategies(
stop=[TranscriptionUserTurnStopStrategy(timeout=TRANSCRIPTION_TIMEOUT)],
stop=[
SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=TRANSCRIPTION_TIMEOUT)
],
),
user_turn_stop_timeout=USER_TURN_STOP_TIMEOUT,
),
@@ -270,13 +283,13 @@ class TestLLMUserAggregator(unittest.IsolatedAsyncioTestCase):
pipeline = Pipeline([user_aggregator])
# Transcript arrives before VAD stop, then we wait for user_speech_timeout
frames_to_send = [
VADUserStartedSpeakingFrame(),
VADUserStoppedSpeakingFrame(),
SleepFrame(sleep=USER_TURN_STOP_TIMEOUT - 0.1),
TranscriptionFrame(text="Hello!", user_id="", timestamp="now"),
SleepFrame(sleep=USER_TURN_STOP_TIMEOUT - 0.1),
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT),
VADUserStoppedSpeakingFrame(),
# Wait for user_speech_timeout (TRANSCRIPTION_TIMEOUT=0.1s) to elapse
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT + 0.05),
]
await run_test(
pipeline,

123
tests/test_service_switcher.py
View File

@@ -12,6 +12,9 @@ from dataclasses import dataclass
from pipecat.frames.frames import (
Frame,
ManuallySwitchServiceFrame,
RequestMetadataFrame,
ServiceMetadataFrame,
StartFrame,
SystemFrame,
TextFrame,
)
@@ -54,6 +57,47 @@ class MockFrameProcessor(FrameProcessor):
self.frame_count = 0
@dataclass
class MockMetadataFrame(ServiceMetadataFrame):
"""A mock metadata frame for testing ServiceMetadataFrame handling."""
pass
class MockMetadataService(FrameProcessor):
"""A mock service that emits ServiceMetadataFrame like STT services.
Pushes MockMetadataFrame on StartFrame and RequestMetadataFrame.
"""
def __init__(self, test_name: str, **kwargs):
super().__init__(name=test_name, **kwargs)
self.test_name = test_name
self.processed_frames = []
self.metadata_push_count = 0
async def process_frame(self, frame: Frame, direction: FrameDirection):
await super().process_frame(frame, direction)
self.processed_frames.append(frame)
if isinstance(frame, StartFrame):
await self.push_frame(frame, direction)
await self._push_metadata()
elif isinstance(frame, RequestMetadataFrame):
# Don't push RequestMetadataFrame downstream (it's internal)
await self._push_metadata()
else:
await self.push_frame(frame, direction)
async def _push_metadata(self):
self.metadata_push_count += 1
await self.push_frame(MockMetadataFrame(service_name=self.test_name))
def reset_counters(self):
self.processed_frames = []
self.metadata_push_count = 0
@dataclass
class DummySystemFrame(SystemFrame):
"""A dummy system frame for testing purposes."""
@@ -336,5 +380,84 @@ class TestServiceSwitcher(unittest.IsolatedAsyncioTestCase):
self.assertEqual(switcher2_service2_texts[0].text, "After switching second switcher")
class TestServiceSwitcherMetadata(unittest.IsolatedAsyncioTestCase):
"""Test cases for ServiceMetadataFrame handling in ServiceSwitcher."""
def setUp(self):
"""Set up test fixtures with mock metadata services."""
self.service1 = MockMetadataService("service1")
self.service2 = MockMetadataService("service2")
self.services = [self.service1, self.service2]
async def test_only_active_service_metadata_at_startup(self):
"""Test that only the active service's metadata leaves the ServiceSwitcher at startup."""
switcher = ServiceSwitcher(self.services, ServiceSwitcherStrategyManual)
# Run the pipeline (StartFrame triggers metadata emission)
output_frames = []
async def capture_frame(frame: Frame):
output_frames.append(frame)
await run_test(
switcher,
frames_to_send=[TextFrame(text="test")],
expected_down_frames=[MockMetadataFrame, TextFrame],
expected_up_frames=[],
)
# Both services push metadata internally on StartFrame, but only the
# active service's metadata passes through the filter
self.assertEqual(self.service1.metadata_push_count, 1) # StartFrame (passes filter)
self.assertEqual(self.service2.metadata_push_count, 1) # StartFrame (blocked by filter)
async def test_metadata_emitted_on_service_switch(self):
"""Test that switching services triggers metadata emission from the new active service."""
switcher = ServiceSwitcher(self.services, ServiceSwitcherStrategyManual)
# Reset counters after startup
self.service1.reset_counters()
self.service2.reset_counters()
await run_test(
switcher,
frames_to_send=[
TextFrame(text="before switch"),
ManuallySwitchServiceFrame(service=self.service2),
TextFrame(text="after switch"),
],
expected_down_frames=[
MockMetadataFrame, # From startup (service1)
TextFrame,
ManuallySwitchServiceFrame,
TextFrame,
MockMetadataFrame, # From service2 after switch
],
expected_up_frames=[],
)
# service2 should have received RequestMetadataFrame after becoming active
request_frames = [
f for f in self.service2.processed_frames if isinstance(f, RequestMetadataFrame)
]
self.assertEqual(len(request_frames), 1)
async def test_inactive_service_metadata_blocked(self):
"""Test that metadata from inactive services is blocked."""
switcher = ServiceSwitcher(self.services, ServiceSwitcherStrategyManual)
# Run and collect output frames
await run_test(
switcher,
frames_to_send=[TextFrame(text="test")],
expected_down_frames=[MockMetadataFrame, TextFrame],
expected_up_frames=[],
)
# service2 pushed metadata on StartFrame, but it should have been blocked
self.assertGreaterEqual(self.service2.metadata_push_count, 1)
# Only one MockMetadataFrame should have left (from service1)
if __name__ == "__main__":
unittest.main()

10
tests/test_user_turn_controller.py
View File

@@ -18,11 +18,13 @@ from pipecat.frames.frames import (
from pipecat.turns.user_start.min_words_user_turn_start_strategy import (
MinWordsUserTurnStartStrategy,
)
from pipecat.turns.user_stop import SpeechTimeoutUserTurnStopStrategy
from pipecat.turns.user_turn_controller import UserTurnController
from pipecat.turns.user_turn_strategies import ExternalUserTurnStrategies, UserTurnStrategies
from pipecat.utils.asyncio.task_manager import TaskManager, TaskManagerParams
USER_TURN_STOP_TIMEOUT = 0.2
TRANSCRIPTION_TIMEOUT = 0.1
class TestUserTurnController(unittest.IsolatedAsyncioTestCase):
@@ -31,7 +33,11 @@ class TestUserTurnController(unittest.IsolatedAsyncioTestCase):
self.task_manager.setup(TaskManagerParams(loop=asyncio.get_running_loop()))
async def test_default_user_turn_strategies(self):
controller = UserTurnController(user_turn_strategies=UserTurnStrategies())
controller = UserTurnController(
user_turn_strategies=UserTurnStrategies(
stop=[SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=TRANSCRIPTION_TIMEOUT)],
)
)
await controller.setup(self.task_manager)
@@ -60,6 +66,8 @@ class TestUserTurnController(unittest.IsolatedAsyncioTestCase):
await controller.process_frame(VADUserStoppedSpeakingFrame())
self.assertTrue(should_start)
# Wait for user_speech_timeout to elapse
await asyncio.sleep(TRANSCRIPTION_TIMEOUT + 0.1)
self.assertTrue(should_stop)
async def test_user_turn_start_reset(self):

20
tests/test_user_turn_processor.py
View File

@@ -16,7 +16,7 @@ from pipecat.frames.frames import (
)
from pipecat.pipeline.pipeline import Pipeline
from pipecat.tests.utils import SleepFrame, run_test
from pipecat.turns.user_stop import TranscriptionUserTurnStopStrategy
from pipecat.turns.user_stop import SpeechTimeoutUserTurnStopStrategy
from pipecat.turns.user_turn_processor import UserTurnProcessor
from pipecat.turns.user_turn_strategies import UserTurnStrategies
@@ -26,7 +26,11 @@ TRANSCRIPTION_TIMEOUT = 0.1
class TestUserTurnProcessor(unittest.IsolatedAsyncioTestCase):
async def test_default_user_turn_strategies(self):
user_turn_processor = UserTurnProcessor(user_turn_strategies=UserTurnStrategies())
user_turn_processor = UserTurnProcessor(
user_turn_strategies=UserTurnStrategies(
stop=[SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=TRANSCRIPTION_TIMEOUT)],
)
)
should_start = None
should_stop = None
@@ -48,6 +52,8 @@ class TestUserTurnProcessor(unittest.IsolatedAsyncioTestCase):
TranscriptionFrame(text="Hello!", user_id="", timestamp="now"),
SleepFrame(),
VADUserStoppedSpeakingFrame(),
# Wait for user_speech_timeout to elapse
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT + 0.1),
]
expected_down_frames = [
VADUserStartedSpeakingFrame,
@@ -109,7 +115,7 @@ class TestUserTurnProcessor(unittest.IsolatedAsyncioTestCase):
async def test_user_turn_stop_timeout_transcription(self):
user_turn_processor = UserTurnProcessor(
user_turn_strategies=UserTurnStrategies(
stop=[TranscriptionUserTurnStopStrategy(timeout=TRANSCRIPTION_TIMEOUT)],
stop=[SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=TRANSCRIPTION_TIMEOUT)],
),
user_turn_stop_timeout=USER_TURN_STOP_TIMEOUT,
)
@@ -135,13 +141,13 @@ class TestUserTurnProcessor(unittest.IsolatedAsyncioTestCase):
pipeline = Pipeline([user_turn_processor])
# Transcript arrives before VAD stop, then we wait for user_speech_timeout
frames_to_send = [
VADUserStartedSpeakingFrame(),
VADUserStoppedSpeakingFrame(),
SleepFrame(sleep=USER_TURN_STOP_TIMEOUT - 0.1),
TranscriptionFrame(text="Hello!", user_id="", timestamp="now"),
SleepFrame(sleep=USER_TURN_STOP_TIMEOUT - 0.1),
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT),
VADUserStoppedSpeakingFrame(),
# Wait for user_speech_timeout (TRANSCRIPTION_TIMEOUT=0.1s) to elapse
SleepFrame(sleep=TRANSCRIPTION_TIMEOUT + 0.05),
]
await run_test(
pipeline,

108
tests/test_user_turn_stop_strategy.py
View File

@@ -9,25 +9,38 @@ import unittest
from pipecat.frames.frames import (
InterimTranscriptionFrame,
STTMetadataFrame,
TranscriptionFrame,
UserStartedSpeakingFrame,
UserStoppedSpeakingFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.turns.user_stop import ExternalUserTurnStopStrategy, TranscriptionUserTurnStopStrategy
from pipecat.turns.user_stop import ExternalUserTurnStopStrategy, SpeechTimeoutUserTurnStopStrategy
from pipecat.utils.asyncio.task_manager import TaskManager, TaskManagerParams
AGGREGATION_TIMEOUT = 0.1
# Use 0 STT timeout for deterministic test timing
STT_TIMEOUT = 0.0
class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
class TestSpeechTimeoutUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None:
self.task_manager = TaskManager()
self.task_manager.setup(TaskManagerParams(loop=asyncio.get_running_loop()))
async def _create_strategy(self, user_speech_timeout=AGGREGATION_TIMEOUT):
"""Create strategy and configure STT timeout via metadata frame."""
strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=user_speech_timeout)
await strategy.setup(self.task_manager)
# Set STT timeout via metadata frame (as would happen in real pipeline)
await strategy.process_frame(
STTMetadataFrame(service_name="test", ttfs_p99_latency=STT_TIMEOUT)
)
return strategy
async def test_ste(self):
strategy = TranscriptionUserTurnStopStrategy()
strategy = await self._create_strategy()
should_start = None
@@ -46,13 +59,15 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
# E
await strategy.process_frame(VADUserStoppedSpeakingFrame())
self.assertIsNone(should_start)
# Transcription comes in between user started/stopped and there are not
# interim, we just trigger bot speech.
# Transcription came in between user started/stopped. Now we wait for
# timeout before triggering.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_site(self):
strategy = TranscriptionUserTurnStopStrategy()
strategy = await self._create_strategy()
should_start = None
@@ -77,13 +92,15 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
# E
await strategy.process_frame(VADUserStoppedSpeakingFrame())
self.assertIsNone(should_start)
# Transcription comes in between user started/stopped, so we trigger
# speech right away.
# Transcription came in between user started/stopped. Now we wait for
# timeout before triggering.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_st1iest2e(self):
strategy = TranscriptionUserTurnStopStrategy()
strategy = await self._create_strategy()
should_start = None
@@ -122,15 +139,14 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
# E
await strategy.process_frame(VADUserStoppedSpeakingFrame())
self.assertIsNone(should_start)
# There was an interim before the first user stopped speaking, then we
# got a transcription comes in between user started/stopped, so we
# trigger speech right away.
# Now we wait for timeout before triggering.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_siet(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -163,8 +179,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
self.assertTrue(should_start)
async def test_sieit(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -205,8 +220,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
self.assertTrue(should_start)
async def test_set(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -235,8 +249,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
self.assertTrue(should_start)
async def test_seit(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -271,8 +284,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
self.assertTrue(should_start)
async def test_st1et2(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -291,26 +303,37 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
# E
await strategy.process_frame(VADUserStoppedSpeakingFrame())
self.assertIsNone(should_start)
# Transcription comes between user start/stopped speaking, we need to
# trigger speech right away.
# Transcription came between user start/stopped speaking, wait for timeout.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
should_start = None
# Reset for next turn (in real usage, UserTurnController would do this)
await strategy.reset()
# S - new turn starts
await strategy.process_frame(VADUserStartedSpeakingFrame())
self.assertIsNone(should_start)
# T2
await strategy.process_frame(
TranscriptionFrame(text="How are you?", user_id="cat", timestamp="")
)
self.assertIsNone(should_start)
# E
await strategy.process_frame(VADUserStoppedSpeakingFrame())
self.assertIsNone(should_start)
# Transcription comes after user stopped speaking, we need to wait for
# at least the aggregation timeout.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_set1t2(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -343,8 +366,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
self.assertTrue(should_start)
async def test_siet1it2(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -388,8 +410,8 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
self.assertTrue(should_start)
async def test_t(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
"""Transcription without VAD - uses fallback timeout."""
strategy = await self._create_strategy()
should_start = None
@@ -402,14 +424,13 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp=""))
self.assertIsNone(should_start)
# Transcription comes after user stopped speaking, we need to wait for
# at least the aggregation timeout.
# Transcription without VAD triggers fallback timeout.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_it(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
"""Interim + Transcription without VAD - uses fallback timeout."""
strategy = await self._create_strategy()
should_start = None
@@ -427,14 +448,12 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp=""))
self.assertIsNone(should_start)
# Transcription comes after user stopped speaking, we need to wait for
# at least the aggregation timeout.
# Transcription without VAD triggers fallback timeout.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_sie_delay_it(self):
strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
strategy = await self._create_strategy()
should_start = None
@@ -456,23 +475,22 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
await strategy.process_frame(VADUserStoppedSpeakingFrame())
self.assertIsNone(should_start)
# Delay
# Delay - timeout expires but no transcript yet
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
# Still no trigger because no transcript received
self.assertIsNone(should_start)
# I
await strategy.process_frame(
InterimTranscriptionFrame(text="How", user_id="cat", timestamp="")
)
# T
# T (finalized) - triggers immediately since timeout already elapsed
await strategy.process_frame(
TranscriptionFrame(text="How are you?", user_id="cat", timestamp="")
TranscriptionFrame(text="How are you?", user_id="cat", timestamp="", finalized=True)
)
self.assertIsNone(should_start)
# Transcription comes after user stopped speaking, we need to wait for
# at least the aggregation timeout.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
# Finalized transcript received after timeout, triggers immediately
self.assertTrue(should_start)