Improve user turn stop timing by triggering timeout from VAD stop

Refactor TranscriptionUserTurnStopStrategy and TurnAnalyzerUserTurnStopStrategy
to use VADUserStoppedSpeakingFrame as the ground truth for when speech ended,
rather than triggering timeouts from transcription frames.

tests/test_user_turn_stop_strategy.py

@@ -9,25 +9,38 @@ import unittest
 
 from pipecat.frames.frames import (
     InterimTranscriptionFrame,
+    STTMetadataFrame,
     TranscriptionFrame,
     UserStartedSpeakingFrame,
     UserStoppedSpeakingFrame,
     VADUserStartedSpeakingFrame,
     VADUserStoppedSpeakingFrame,
 )
-from pipecat.turns.user_stop import ExternalUserTurnStopStrategy, TranscriptionUserTurnStopStrategy
+from pipecat.turns.user_stop import ExternalUserTurnStopStrategy, SpeechTimeoutUserTurnStopStrategy
 from pipecat.utils.asyncio.task_manager import TaskManager, TaskManagerParams
 
 AGGREGATION_TIMEOUT = 0.1
+# Use 0 STT timeout for deterministic test timing
+STT_TIMEOUT = 0.0
 
 
-class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
+class TestSpeechTimeoutUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
     async def asyncSetUp(self) -> None:
         self.task_manager = TaskManager()
         self.task_manager.setup(TaskManagerParams(loop=asyncio.get_running_loop()))
 
+    async def _create_strategy(self, user_speech_timeout=AGGREGATION_TIMEOUT):
+        """Create strategy and configure STT timeout via metadata frame."""
+        strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=user_speech_timeout)
+        await strategy.setup(self.task_manager)
+        # Set STT timeout via metadata frame (as would happen in real pipeline)
+        await strategy.process_frame(
+            STTMetadataFrame(service_name="test", ttfs_p99_latency=STT_TIMEOUT)
+        )
+        return strategy
+
     async def test_ste(self):
-        strategy = TranscriptionUserTurnStopStrategy()
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -46,13 +59,15 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
 
         # E
         await strategy.process_frame(VADUserStoppedSpeakingFrame())
+        self.assertIsNone(should_start)
 
-        # Transcription comes in between user started/stopped and there are not
-        # interim, we just trigger bot speech.
+        # Transcription came in between user started/stopped. Now we wait for
+        # timeout before triggering.
+        await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
         self.assertTrue(should_start)
 
     async def test_site(self):
-        strategy = TranscriptionUserTurnStopStrategy()
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -77,13 +92,15 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
 
         # E
         await strategy.process_frame(VADUserStoppedSpeakingFrame())
+        self.assertIsNone(should_start)
 
-        # Transcription comes in between user started/stopped, so we trigger
-        # speech right away.
+        # Transcription came in between user started/stopped. Now we wait for
+        # timeout before triggering.
+        await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
         self.assertTrue(should_start)
 
     async def test_st1iest2e(self):
-        strategy = TranscriptionUserTurnStopStrategy()
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -122,15 +139,14 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
 
         # E
         await strategy.process_frame(VADUserStoppedSpeakingFrame())
+        self.assertIsNone(should_start)
 
-        # There was an interim before the first user stopped speaking, then we
-        # got a transcription comes in between user started/stopped, so we
-        # trigger speech right away.
+        # Now we wait for timeout before triggering.
+        await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
         self.assertTrue(should_start)
 
     async def test_siet(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -163,8 +179,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         self.assertTrue(should_start)
 
     async def test_sieit(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -205,8 +220,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         self.assertTrue(should_start)
 
     async def test_set(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -235,8 +249,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         self.assertTrue(should_start)
 
     async def test_seit(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -271,8 +284,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         self.assertTrue(should_start)
 
     async def test_st1et2(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -291,26 +303,37 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
 
         # E
         await strategy.process_frame(VADUserStoppedSpeakingFrame())
+        self.assertIsNone(should_start)
 
-        # Transcription comes between user start/stopped speaking, we need to
-        # trigger speech right away.
+        # Transcription came between user start/stopped speaking, wait for timeout.
+        await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
         self.assertTrue(should_start)
         should_start = None
 
+        # Reset for next turn (in real usage, UserTurnController would do this)
+        await strategy.reset()
+
+        # S - new turn starts
+        await strategy.process_frame(VADUserStartedSpeakingFrame())
+        self.assertIsNone(should_start)
+
         # T2
         await strategy.process_frame(
             TranscriptionFrame(text="How are you?", user_id="cat", timestamp="")
         )
         self.assertIsNone(should_start)
 
+        # E
+        await strategy.process_frame(VADUserStoppedSpeakingFrame())
+        self.assertIsNone(should_start)
+
         # Transcription comes after user stopped speaking, we need to wait for
         # at least the aggregation timeout.
         await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
         self.assertTrue(should_start)
 
     async def test_set1t2(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -343,8 +366,7 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         self.assertTrue(should_start)
 
     async def test_siet1it2(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -388,8 +410,8 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         self.assertTrue(should_start)
 
     async def test_t(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        """Transcription without VAD - uses fallback timeout."""
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -402,14 +424,13 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp=""))
         self.assertIsNone(should_start)
 
-        # Transcription comes after user stopped speaking, we need to wait for
-        # at least the aggregation timeout.
+        # Transcription without VAD triggers fallback timeout.
         await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
         self.assertTrue(should_start)
 
     async def test_it(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        """Interim + Transcription without VAD - uses fallback timeout."""
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -427,14 +448,12 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp=""))
         self.assertIsNone(should_start)
 
-        # Transcription comes after user stopped speaking, we need to wait for
-        # at least the aggregation timeout.
+        # Transcription without VAD triggers fallback timeout.
         await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
         self.assertTrue(should_start)
 
     async def test_sie_delay_it(self):
-        strategy = TranscriptionUserTurnStopStrategy(timeout=AGGREGATION_TIMEOUT)
-        await strategy.setup(self.task_manager)
+        strategy = await self._create_strategy()
 
         should_start = None
 
@@ -456,23 +475,22 @@ class TestTranscriptionUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
         await strategy.process_frame(VADUserStoppedSpeakingFrame())
         self.assertIsNone(should_start)
 
-        # Delay
+        # Delay - timeout expires but no transcript yet
         await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
+        # Still no trigger because no transcript received
+        self.assertIsNone(should_start)
 
         # I
         await strategy.process_frame(
             InterimTranscriptionFrame(text="How", user_id="cat", timestamp="")
         )
 
-        # T
+        # T (finalized) - triggers immediately since timeout already elapsed
         await strategy.process_frame(
-            TranscriptionFrame(text="How are you?", user_id="cat", timestamp="")
+            TranscriptionFrame(text="How are you?", user_id="cat", timestamp="", finalized=True)
         )
-        self.assertIsNone(should_start)
 
-        # Transcription comes after user stopped speaking, we need to wait for
-        # at least the aggregation timeout.
-        await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
+        # Finalized transcript received after timeout, triggers immediately
         self.assertTrue(should_start)