diff --git a/changelog/3562.changed.md b/changelog/3562.changed.md new file mode 100644 index 000000000..533f37b2f --- /dev/null +++ b/changelog/3562.changed.md @@ -0,0 +1,2 @@ +- Added support for TTFS in `SpeechmaticsSTTService` and set the default mode to `EXTERNAL` to support Pipecat-controlled VAD. +- Changed dependency to `speechmatics-voice[smart]>=0.2.8` diff --git a/pyproject.toml b/pyproject.toml index 0cb985f34..884b5afd1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -109,7 +109,7 @@ silero = [ "onnxruntime>=1.20.1,<2" ] simli = [ "simli-ai~=1.0.3"] soniox = [ "pipecat-ai[websockets-base]" ] soundfile = [ "soundfile~=0.13.1" ] -speechmatics = [ "speechmatics-voice[smart]>=0.2.6" ] +speechmatics = [ "speechmatics-voice[smart]>=0.2.8" ] strands = [ "strands-agents>=1.9.1,<2" ] tavus=[] together = [] diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index d12b2a7d3..41434addf 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -66,7 +66,7 @@ class TurnDetectionMode(str, Enum): """Endpoint and turn detection handling mode. How the STT engine handles the endpointing of speech. If using Pipecat's built-in endpointing, - then use `TurnDetectionMode.FIXED` (default). + then use `TurnDetectionMode.EXTERNAL` (default). To use the STT engine's built-in endpointing, then use `TurnDetectionMode.ADAPTIVE` for simple voice activity detection or `TurnDetectionMode.SMART_TURN` for more advanced ML-based @@ -106,7 +106,7 @@ class SpeechmaticsSTTService(STTService): turn_detection_mode: Endpoint handling, one of `TurnDetectionMode.FIXED`, `TurnDetectionMode.EXTERNAL`, `TurnDetectionMode.ADAPTIVE` and - `TurnDetectionMode.SMART_TURN`. Defaults to `TurnDetectionMode.FIXED`. + `TurnDetectionMode.SMART_TURN`. Defaults to `TurnDetectionMode.EXTERNAL`. speaker_active_format: Formatter for active speaker ID. This formatter is used to format the text output for individual speakers and ensures that the context is clear for @@ -200,6 +200,7 @@ class SpeechmaticsSTTService(STTService): extra_params: Extra parameters to pass to the STT engine. This is a dictionary of additional parameters that can be used to configure the STT engine. Default to None. + """ # Service configuration @@ -207,7 +208,7 @@ class SpeechmaticsSTTService(STTService): language: Language | str = Language.EN # Endpointing mode - turn_detection_mode: TurnDetectionMode = TurnDetectionMode.FIXED + turn_detection_mode: TurnDetectionMode = TurnDetectionMode.EXTERNAL # Output formatting speaker_active_format: str | None = None @@ -345,7 +346,7 @@ class SpeechmaticsSTTService(STTService): params.speaker_passive_format or params.speaker_active_format ) - # Metrics + # Model + metrics self.set_model_name(self._config.operating_point.value) # Message queue @@ -691,6 +692,7 @@ class SpeechmaticsSTTService(STTService): f"{self} VADUserStoppedSpeakingFrame received but internal VAD is being used" ) elif not self._enable_vad and self._client is not None: + self.request_finalize() self._client.finalize() async def _send_frames(self, segments: list[dict[str, Any]], finalized: bool = False) -> None: @@ -734,16 +736,33 @@ class SpeechmaticsSTTService(STTService): # If final, then re-parse into TranscriptionFrame if finalized: + # Do any segments have `is_eou` set to True? + if ( + any(segment.get("is_eou", False) for segment in segments) + and self._finalize_requested + ): + self.confirm_finalize() + + # Add the finalized frames frames += [TranscriptionFrame(**attr_from_segment(segment)) for segment in segments] + + # Handle the text (for metrics reporting) finalized_text = "|".join([s["text"] for s in segments]) - await self._handle_transcription(finalized_text, True, segments[0]["language"]) + await self._handle_transcription( + finalized_text, is_final=True, language=segments[0]["language"] + ) + + # Log the frames logger.debug(f"{self} finalized transcript: {[f.text for f in frames]}") # Return as interim results (unformatted) else: + # Add the interim frames frames += [ InterimTranscriptionFrame(**attr_from_segment(segment)) for segment in segments ] + + # Log the frames logger.debug(f"{self} interim transcript: {[f.text for f in frames]}") # Send the frames