Merge pull request #3562 from speechmatics/fix/smx-ttfs-finals

Support TTFS for Speechmatics STT

changelog/3562.changed.md

@@ -0,0 +1,2 @@
+- Added support for TTFS in `SpeechmaticsSTTService` and set the default mode to `EXTERNAL` to support Pipecat-controlled VAD.
+- Changed dependency to `speechmatics-voice[smart]>=0.2.8`

pyproject.toml

@@ -109,7 +109,7 @@ silero = [ "onnxruntime>=1.20.1,<2" ]
 simli = [ "simli-ai~=1.0.3"]
 soniox = [ "pipecat-ai[websockets-base]" ]
 soundfile = [ "soundfile~=0.13.1" ]
-speechmatics = [ "speechmatics-voice[smart]>=0.2.6" ]
+speechmatics = [ "speechmatics-voice[smart]>=0.2.8" ]
 strands = [ "strands-agents>=1.9.1,<2" ]
 tavus=[]
 together = []

src/pipecat/services/speechmatics/stt.py

@@ -66,7 +66,7 @@ class TurnDetectionMode(str, Enum):
     """Endpoint and turn detection handling mode.
 
     How the STT engine handles the endpointing of speech. If using Pipecat's built-in endpointing,
-    then use `TurnDetectionMode.FIXED` (default).
+    then use `TurnDetectionMode.EXTERNAL` (default).
 
     To use the STT engine's built-in endpointing, then use `TurnDetectionMode.ADAPTIVE` for simple
     voice activity detection or `TurnDetectionMode.SMART_TURN` for more advanced ML-based
@@ -106,7 +106,7 @@ class SpeechmaticsSTTService(STTService):
 
             turn_detection_mode: Endpoint handling, one of `TurnDetectionMode.FIXED`,
                 `TurnDetectionMode.EXTERNAL`, `TurnDetectionMode.ADAPTIVE` and
-                `TurnDetectionMode.SMART_TURN`. Defaults to `TurnDetectionMode.FIXED`.
+                `TurnDetectionMode.SMART_TURN`. Defaults to `TurnDetectionMode.EXTERNAL`.
 
             speaker_active_format: Formatter for active speaker ID. This formatter is used to format
                 the text output for individual speakers and ensures that the context is clear for
@@ -200,6 +200,7 @@ class SpeechmaticsSTTService(STTService):
             extra_params: Extra parameters to pass to the STT engine. This is a dictionary of
                 additional parameters that can be used to configure the STT engine.
                 Default to None.
+
         """
 
         # Service configuration
@@ -207,7 +208,7 @@ class SpeechmaticsSTTService(STTService):
         language: Language | str = Language.EN
 
         # Endpointing mode
-        turn_detection_mode: TurnDetectionMode = TurnDetectionMode.FIXED
+        turn_detection_mode: TurnDetectionMode = TurnDetectionMode.EXTERNAL
 
         # Output formatting
         speaker_active_format: str | None = None
@@ -345,7 +346,7 @@ class SpeechmaticsSTTService(STTService):
             params.speaker_passive_format or params.speaker_active_format
         )
 
-        # Metrics
+        # Model + metrics
         self.set_model_name(self._config.operating_point.value)
 
         # Message queue
@@ -691,6 +692,7 @@ class SpeechmaticsSTTService(STTService):
                     f"{self} VADUserStoppedSpeakingFrame received but internal VAD is being used"
                 )
             elif not self._enable_vad and self._client is not None:
+                self.request_finalize()
                 self._client.finalize()
 
     async def _send_frames(self, segments: list[dict[str, Any]], finalized: bool = False) -> None:
@@ -734,16 +736,33 @@ class SpeechmaticsSTTService(STTService):
 
         # If final, then re-parse into TranscriptionFrame
         if finalized:
+            # Do any segments have `is_eou` set to True?
+            if (
+                any(segment.get("is_eou", False) for segment in segments)
+                and self._finalize_requested
+            ):
+                self.confirm_finalize()
+
+            # Add the finalized frames
             frames += [TranscriptionFrame(**attr_from_segment(segment)) for segment in segments]
+
+            # Handle the text (for metrics reporting)
             finalized_text = "|".join([s["text"] for s in segments])
-            await self._handle_transcription(finalized_text, True, segments[0]["language"])
+            await self._handle_transcription(
+                finalized_text, is_final=True, language=segments[0]["language"]
+            )
+
+            # Log the frames
             logger.debug(f"{self} finalized transcript: {[f.text for f in frames]}")
 
         # Return as interim results (unformatted)
         else:
+            # Add the interim frames
             frames += [
                 InterimTranscriptionFrame(**attr_from_segment(segment)) for segment in segments
             ]
+
+            # Log the frames
             logger.debug(f"{self} interim transcript: {[f.text for f in frames]}")
 
         # Send the frames