wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

TranscriptionBotTurnStartStrategy: improve by using interim transcriptions

Browse Source
This commit is contained in:
Aleix Conchillo Flaqué
2025-12-08 18:22:02 -08:00
parent 355fcf3282
commit 76561da850
1 changed files with 17 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/pipecat/turns/bot/transcription_bot_turn_start_strategy.py
View File

@@ -11,6 +11,7 @@ from typing import Optional
from pipecat.frames.frames import ( from pipecat.frames.frames import (
Frame, Frame,
InterimTranscriptionFrame,
TranscriptionFrame, TranscriptionFrame,
VADUserStartedSpeakingFrame, VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame, VADUserStoppedSpeakingFrame,
@@ -38,6 +39,7 @@ class TranscriptionBotTurnStartStrategy(BaseBotTurnStartStrategy):
self._timeout = timeout self._timeout = timeout
self._text = "" self._text = ""
self._vad_user_speaking = False self._vad_user_speaking = False
self._seen_interim_results = False
self._event = asyncio.Event() self._event = asyncio.Event()
self._task: Optional[asyncio.Task] = None self._task: Optional[asyncio.Task] = None
@@ -46,6 +48,7 @@ class TranscriptionBotTurnStartStrategy(BaseBotTurnStartStrategy):
await super().reset() await super().reset()
self._text = "" self._text = ""
self._vad_user_speaking = False self._vad_user_speaking = False
self._seen_interim_results = False
self._event.clear() self._event.clear()
async def setup(self, task_manager: BaseTaskManager): async def setup(self, task_manager: BaseTaskManager):
@@ -78,6 +81,8 @@ class TranscriptionBotTurnStartStrategy(BaseBotTurnStartStrategy):
await self._handle_vad_user_started_speaking(frame) await self._handle_vad_user_started_speaking(frame)
elif isinstance(frame, VADUserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
await self._handle_vad_user_stopped_speaking(frame) await self._handle_vad_user_stopped_speaking(frame)
elif isinstance(frame, InterimTranscriptionFrame):
await self._handle_interim_transcription(frame)
elif isinstance(frame, TranscriptionFrame): elif isinstance(frame, TranscriptionFrame):
await self._handle_transcription(frame) await self._handle_transcription(frame)
@@ -88,10 +93,17 @@ class TranscriptionBotTurnStartStrategy(BaseBotTurnStartStrategy):
async def _handle_vad_user_stopped_speaking(self, _: VADUserStoppedSpeakingFrame): async def _handle_vad_user_stopped_speaking(self, _: VADUserStoppedSpeakingFrame):
"""Handle when the VAD indicates the user has stopped speaking.""" """Handle when the VAD indicates the user has stopped speaking."""
self._vad_user_speaking = False self._vad_user_speaking = False
await self._maybe_trigger_bot_turn_started()
async def _handle_interim_transcription(self, frame: InterimTranscriptionFrame):
self._seen_interim_results = True
async def _handle_transcription(self, frame: TranscriptionFrame): async def _handle_transcription(self, frame: TranscriptionFrame):
"""Handle user transcription.""" """Handle user transcription."""
self._text += frame.text self._text += frame.text
# We just got a final result, so let's reset interim results.
self._seen_interim_results = False
# Reset aggregation timer.
self._event.set() self._event.set()
async def _task_handler(self): async def _task_handler(self):
@@ -107,5 +119,8 @@ class TranscriptionBotTurnStartStrategy(BaseBotTurnStartStrategy):
await asyncio.wait_for(self._event.wait(), timeout=self._timeout) await asyncio.wait_for(self._event.wait(), timeout=self._timeout)
self._event.clear() self._event.clear()
except asyncio.TimeoutError: except asyncio.TimeoutError:
if self._text and not self._vad_user_speaking: await self._maybe_trigger_bot_turn_started()
await self.trigger_bot_turn_started()
async def _maybe_trigger_bot_turn_started(self):
if not self._vad_user_speaking and not self._seen_interim_results and self._text:
await self.trigger_bot_turn_started()