wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix STT services that rely on VAD stop speaking status to finalize the transcript (#3283)

Browse Source 
Updates to AssemblyAISTTService, CartesiaSTTService, DeepgramSageMakerSTTService, DeepgramSTTService to use VADUser*SpeakingFrame
This commit is contained in:
Mark Backman
2025-12-22 12:54:06 -05:00
committed by GitHub
parent 0958c658db
commit 2910b683a4
4 changed files with 16 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/pipecat/services/assemblyai/stt.py
View File

@@ -25,8 +25,8 @@ from pipecat.frames.frames import (
InterimTranscriptionFrame, InterimTranscriptionFrame,
StartFrame, StartFrame,
TranscriptionFrame, TranscriptionFrame,
UserStartedSpeakingFrame, VADUserStartedSpeakingFrame,
UserStoppedSpeakingFrame, VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.stt_service import WebsocketSTTService from pipecat.services.stt_service import WebsocketSTTService
@@ -160,9 +160,9 @@ class AssemblyAISTTService(WebsocketSTTService):
direction: Direction of frame processing. direction: Direction of frame processing.
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, VADUserStartedSpeakingFrame):
await self.start_ttfb_metrics() await self.start_ttfb_metrics()
elif isinstance(frame, UserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
if ( if (
self._vad_force_turn_endpoint self._vad_force_turn_endpoint
and self._websocket and self._websocket

8
src/pipecat/services/cartesia/stt.py
View File

@@ -23,8 +23,8 @@ from pipecat.frames.frames import (
InterimTranscriptionFrame, InterimTranscriptionFrame,
StartFrame, StartFrame,
TranscriptionFrame, TranscriptionFrame,
UserStartedSpeakingFrame, VADUserStartedSpeakingFrame,
UserStoppedSpeakingFrame, VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.stt_service import WebsocketSTTService from pipecat.services.stt_service import WebsocketSTTService
@@ -221,9 +221,9 @@ class CartesiaSTTService(WebsocketSTTService):
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, VADUserStartedSpeakingFrame):
await self.start_metrics() await self.start_metrics()
elif isinstance(frame, UserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
# Send finalize command to flush the transcription session # Send finalize command to flush the transcription session
if self._websocket and self._websocket.state is State.OPEN: if self._websocket and self._websocket.state is State.OPEN:
await self._websocket.send("finalize") await self._websocket.send("finalize")

10
src/pipecat/services/deepgram/stt.py
View File

@@ -6,7 +6,6 @@
"""Deepgram speech-to-text service implementation.""" """Deepgram speech-to-text service implementation."""
import asyncio
from typing import AsyncGenerator, Dict, Optional from typing import AsyncGenerator, Dict, Optional
from loguru import logger from loguru import logger
@@ -14,13 +13,12 @@ from loguru import logger
from pipecat.frames.frames import ( from pipecat.frames.frames import (
CancelFrame, CancelFrame,
EndFrame, EndFrame,
ErrorFrame,
Frame, Frame,
InterimTranscriptionFrame, InterimTranscriptionFrame,
StartFrame, StartFrame,
TranscriptionFrame, TranscriptionFrame,
UserStartedSpeakingFrame, VADUserStartedSpeakingFrame,
UserStoppedSpeakingFrame, VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.stt_service import STTService from pipecat.services.stt_service import STTService
@@ -329,10 +327,10 @@ class DeepgramSTTService(STTService):
""" """
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, UserStartedSpeakingFrame) and not self.vad_enabled: if isinstance(frame, VADUserStartedSpeakingFrame) and not self.vad_enabled:
# Start metrics if Deepgram VAD is disabled & pipeline VAD has detected speech # Start metrics if Deepgram VAD is disabled & pipeline VAD has detected speech
await self.start_metrics() await self.start_metrics()
elif isinstance(frame, UserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
# https://developers.deepgram.com/docs/finalize # https://developers.deepgram.com/docs/finalize
await self._connection.finalize() await self._connection.finalize()
logger.trace(f"Triggered finalize event on: {frame.name=}, {direction=}") logger.trace(f"Triggered finalize event on: {frame.name=}, {direction=}")

8
src/pipecat/services/deepgram/stt_sagemaker.py
View File

@@ -26,8 +26,8 @@ from pipecat.frames.frames import (
InterimTranscriptionFrame, InterimTranscriptionFrame,
StartFrame, StartFrame,
TranscriptionFrame, TranscriptionFrame,
UserStartedSpeakingFrame, VADUserStartedSpeakingFrame,
UserStoppedSpeakingFrame, VADUserStoppedSpeakingFrame,
) )
from pipecat.processors.frame_processor import FrameDirection from pipecat.processors.frame_processor import FrameDirection
from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient from pipecat.services.aws.sagemaker.bidi_client import SageMakerBidiClient
@@ -432,9 +432,9 @@ class DeepgramSageMakerSTTService(STTService):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
# Start metrics when user starts speaking (if VAD is not provided by Deepgram) # Start metrics when user starts speaking (if VAD is not provided by Deepgram)
if isinstance(frame, UserStartedSpeakingFrame): if isinstance(frame, VADUserStartedSpeakingFrame):
await self.start_metrics() await self.start_metrics()
elif isinstance(frame, UserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
# Send finalize message to Deepgram when user stops speaking # Send finalize message to Deepgram when user stops speaking
# This tells Deepgram to flush any remaining audio and return final results # This tells Deepgram to flush any remaining audio and return final results
if self._client and self._client.is_active: if self._client and self._client.is_active: