Simplify STT finalize handling

This commit is contained in:
Mark Backman
2026-01-24 14:54:58 -05:00
parent 680bcaac66
commit e93112e76e
4 changed files with 8 additions and 28 deletions

View File

@@ -221,13 +221,10 @@ class CartesiaSTTService(WebsocketSTTService):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, VADUserStartedSpeakingFrame): if isinstance(frame, VADUserStartedSpeakingFrame):
# Reset finalize state for new utterance
self.set_finalize_pending(False)
await self._start_metrics() await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
# Send finalize command to flush the transcription session # Send finalize command to flush the transcription session
if self._websocket and self._websocket.state is State.OPEN: if self._websocket and self._websocket.state is State.OPEN:
self.set_finalize_pending(True)
await self._websocket.send("finalize") await self._websocket.send("finalize")
async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]: async def run_stt(self, audio: bytes) -> AsyncGenerator[Frame, None]:

View File

@@ -551,8 +551,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
await super().process_frame(frame, direction) await super().process_frame(frame, direction)
if isinstance(frame, VADUserStartedSpeakingFrame): if isinstance(frame, VADUserStartedSpeakingFrame):
# Reset finalize state for new utterance
self.set_finalize_pending(False)
# Start metrics when user starts speaking # Start metrics when user starts speaking
await self._start_metrics() await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
@@ -560,8 +558,6 @@ class ElevenLabsRealtimeSTTService(WebsocketSTTService):
if self._params.commit_strategy == CommitStrategy.MANUAL: if self._params.commit_strategy == CommitStrategy.MANUAL:
if self._websocket and self._websocket.state is State.OPEN: if self._websocket and self._websocket.state is State.OPEN:
try: try:
# Mark that the next committed transcript should be finalized
self.set_finalize_pending(True)
commit_message = { commit_message = {
"message_type": "input_audio_chunk", "message_type": "input_audio_chunk",
"audio_base_64": "", "audio_base_64": "",

View File

@@ -193,11 +193,9 @@ class SarvamSTTService(STTService):
# Only handle VAD frames when not using Sarvam's VAD signals # Only handle VAD frames when not using Sarvam's VAD signals
if not self._vad_signals: if not self._vad_signals:
if isinstance(frame, VADUserStartedSpeakingFrame): if isinstance(frame, VADUserStartedSpeakingFrame):
self.set_finalize_pending(False)
await self._start_metrics() await self._start_metrics()
elif isinstance(frame, VADUserStoppedSpeakingFrame): elif isinstance(frame, VADUserStoppedSpeakingFrame):
if self._socket_client: if self._socket_client:
self.set_finalize_pending(True)
await self._socket_client.flush() await self._socket_client.flush()
async def set_language(self, language: Language): async def set_language(self, language: Language):

View File

@@ -119,28 +119,15 @@ class STTService(AIService):
""" """
return self._muted return self._muted
def set_finalize_pending(self, value: bool):
"""Set whether the next TranscriptionFrame should be marked as finalized.
When True, the next TranscriptionFrame pushed will have its `finalized`
field set to True, and this flag will automatically reset to False.
This is used to signal that a transcript is the final result for an
utterance, enabling immediate TTFB reporting.
Args:
value: True to mark the next transcription as finalized.
"""
self._finalize_pending = value
def request_finalize(self): def request_finalize(self):
"""Mark that a finalize request has been sent, awaiting server confirmation. """Mark that a finalize request has been sent, awaiting server confirmation.
For providers that require server confirmation before marking transcripts For providers that have explicit server confirmation of finalization
as finalized (e.g., Deepgram's from_finalize field), call this when sending (e.g., Deepgram's from_finalize field), call this when sending the finalize
the finalize request. Then call confirm_finalize() when the server confirms. request. Then call confirm_finalize() when the server confirms.
This is an alternative to set_finalize_pending() for providers that need For providers without server confirmation, don't call this method - just
two-step finalization. send the finalize/flush/commit command and rely on the TTFB timeout.
""" """
self._finalize_requested = True self._finalize_requested = True
@@ -298,7 +285,7 @@ class STTService(AIService):
"""Push a frame downstream, tracking TranscriptionFrame timestamps for TTFB. """Push a frame downstream, tracking TranscriptionFrame timestamps for TTFB.
Stores the timestamp of each TranscriptionFrame for TTFB calculation. Stores the timestamp of each TranscriptionFrame for TTFB calculation.
If the frame is marked as finalized (either directly or via set_finalize_pending), If the frame is marked as finalized (via request_finalize/confirm_finalize),
reports TTFB immediately and cancels any pending timeout. Otherwise, TTFB is reports TTFB immediately and cancels any pending timeout. Otherwise, TTFB is
reported after a timeout. reported after a timeout.
@@ -361,6 +348,7 @@ class STTService(AIService):
"""Handle VAD user started speaking frame to start tracking transcriptions. """Handle VAD user started speaking frame to start tracking transcriptions.
Cancels any pending TTFB timeout, resets TTFB tracking state, and marks user as speaking. Cancels any pending TTFB timeout, resets TTFB tracking state, and marks user as speaking.
Also resets finalization state to prevent stale finalization from a previous utterance.
Args: Args:
frame: The VAD user started speaking frame. frame: The VAD user started speaking frame.
@@ -368,6 +356,7 @@ class STTService(AIService):
await self._reset_stt_ttfb_state() await self._reset_stt_ttfb_state()
self._user_speaking = True self._user_speaking = True
self._finalize_requested = False self._finalize_requested = False
self._finalize_pending = False
async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame): async def _handle_vad_user_stopped_speaking(self, frame: VADUserStoppedSpeakingFrame):
"""Handle VAD user stopped speaking frame. """Handle VAD user stopped speaking frame.