From f7c74cfa8015c74831dbcf6031746e11bb58363e Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 31 Dec 2025 01:28:31 +0000 Subject: [PATCH 1/8] Updated VAD --- src/pipecat/services/speechmatics/stt.py | 26 ++++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index f779f0e77..a628b2f89 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -27,8 +27,8 @@ from pipecat.frames.frames import ( InterimTranscriptionFrame, StartFrame, TranscriptionFrame, - UserStartedSpeakingFrame, - UserStoppedSpeakingFrame, + VADUserStartedSpeakingFrame, + VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.stt_service import STTService @@ -597,17 +597,21 @@ class SpeechmaticsSTTService(STTService): The service will: - Send a BotInterruptionFrame upstream to stop bot speech - - Send a UserStartedSpeakingFrame downstream to notify other components + - Send a VADUserStartedSpeakingFrame downstream to notify other components - Start metrics collection for measuring response times Args: message: the message payload. """ logger.debug(f"{self} StartOfTurn received") - await self.broadcast_frame(UserStartedSpeakingFrame) - await self.push_interruption_task_frame_and_wait() # await self.start_processing_metrics() + # Emit VAD events if enabled + if self._enable_vad: + await self.push_interruption_task_frame_and_wait() + logger.debug(f"{self} sending VADUserStartedSpeakingFrame") + await self.broadcast_frame(VADUserStartedSpeakingFrame) + async def _handle_end_of_turn(self, message: dict[str, Any]) -> None: """Handle EndOfTurn events. @@ -618,14 +622,18 @@ class SpeechmaticsSTTService(STTService): The service will: - Stop processing metrics collection - - Send a UserStoppedSpeakingFrame to signal turn completion + - Send a VADUserStoppedSpeakingFrame to signal turn completion Args: message: the message payload. """ logger.debug(f"{self} EndOfTurn received") # await self.stop_processing_metrics() - await self.broadcast_frame(UserStoppedSpeakingFrame) + + # Emit VAD events if enabled + if self._enable_vad: + logger.debug(f"{self} sending VADUserStoppedSpeakingFrame") + await self.broadcast_frame(VADUserStoppedSpeakingFrame) async def _handle_speakers_result(self, message: dict[str, Any]) -> None: """Handle SpeakersResult events. @@ -660,10 +668,10 @@ class SpeechmaticsSTTService(STTService): self._bot_speaking = False # Force finalization - if isinstance(frame, UserStoppedSpeakingFrame): + if isinstance(frame, VADUserStoppedSpeakingFrame): if self._enable_vad: logger.warning( - f"{self} UserStoppedSpeakingFrame received but internal VAD is being used" + f"{self} VADUserStoppedSpeakingFrame received but internal VAD is being used" ) elif not self._enable_vad and self._client is not None: self._client.finalize() From ba1aeb8f7f725aed76bca35f4120861726c961ea Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 31 Dec 2025 01:31:46 +0000 Subject: [PATCH 2/8] Changelog --- changelog/3328.fixed.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/3328.fixed.md diff --git a/changelog/3328.fixed.md b/changelog/3328.fixed.md new file mode 100644 index 000000000..318425617 --- /dev/null +++ b/changelog/3328.fixed.md @@ -0,0 +1,3 @@ +- Fix to `SpeechmaticsSTTService` for version `0.0.99+`: + - Changed from using `User[Started/Stopped]SpeakingFrame` to `VADUser[Started/Stopped]SpeakingFrame` frames + - Only emit VAD events if VAD is enabled within the plugin (modes other than `TurnDetectionMode.EXTERNAL`) From 8e7a951af875639ca71e3fe07b400e26f4ee9008 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 31 Dec 2025 01:36:58 +0000 Subject: [PATCH 3/8] updated changelog --- changelog/3328.fixed.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/3328.fixed.md b/changelog/3328.fixed.md index 318425617..695ca0045 100644 --- a/changelog/3328.fixed.md +++ b/changelog/3328.fixed.md @@ -1,3 +1,3 @@ - Fix to `SpeechmaticsSTTService` for version `0.0.99+`: - Changed from using `User[Started/Stopped]SpeakingFrame` to `VADUser[Started/Stopped]SpeakingFrame` frames - - Only emit VAD events if VAD is enabled within the plugin (modes other than `TurnDetectionMode.EXTERNAL`) + - Only emit VAD + interruption frames if VAD is enabled within the plugin (modes other than `TurnDetectionMode.EXTERNAL`) From 8203ad08a8ae19dbcbe67c6306dd5d691bb76dcf Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 31 Dec 2025 19:05:29 +0000 Subject: [PATCH 4/8] Updated to have default as FIXED for Pipecat VAD. --- src/pipecat/services/speechmatics/stt.py | 28 +++++++++++++++++++----- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index a628b2f89..38f04cee7 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -46,6 +46,7 @@ try: SpeakerFocusConfig, SpeakerFocusMode, SpeakerIdentifier, + SpeechSegmentConfig, VoiceAgentClient, VoiceAgentConfig, VoiceAgentConfigPreset, @@ -65,13 +66,14 @@ class TurnDetectionMode(str, Enum): """Endpoint and turn detection handling mode. How the STT engine handles the endpointing of speech. If using Pipecat's built-in endpointing, - then use `TurnDetectionMode.EXTERNAL` (default). + then use `TurnDetectionMode.FIXED` (default). To use the STT engine's built-in endpointing, then use `TurnDetectionMode.ADAPTIVE` for simple voice activity detection or `TurnDetectionMode.SMART_TURN` for more advanced ML-based endpointing. """ + FIXED = "fixed" EXTERNAL = "external" ADAPTIVE = "adaptive" SMART_TURN = "smart_turn" @@ -102,9 +104,9 @@ class SpeechmaticsSTTService(STTService): language: Language code for transcription. Defaults to `Language.EN`. - turn_detection_mode: Endpoint handling, one of `TurnDetectionMode.EXTERNAL`, - `TurnDetectionMode.ADAPTIVE` and `TurnDetectionMode.SMART_TURN`. - Defaults to `TurnDetectionMode.EXTERNAL`. + turn_detection_mode: Endpoint handling, one of `TurnDetectionMode.FIXED`, + `TurnDetectionMode.EXTERNAL`, `TurnDetectionMode.ADAPTIVE` and + `TurnDetectionMode.SMART_TURN`. Defaults to `TurnDetectionMode.FIXED`. speaker_active_format: Formatter for active speaker ID. This formatter is used to format the text output for individual speakers and ensures that the context is clear for @@ -177,6 +179,10 @@ class SpeechmaticsSTTService(STTService): speaker activity detection. This setting is used only for the formatted text output of individual segments. + split_sentences: Emit finalized sentences mid-turn. When enabled, as soon as a sentence + is finalized, it will be emitted as a final segment. This is useful for applications + that need to process sentences as they are finalized. Defaults to False. + enable_diarization: Enable speaker diarization. When enabled, the STT engine will determine and attribute words to unique speakers. The speaker_sensitivity parameter can be used to adjust the sensitivity of diarization. @@ -201,7 +207,7 @@ class SpeechmaticsSTTService(STTService): language: Language | str = Language.EN # Endpointing mode - turn_detection_mode: TurnDetectionMode = TurnDetectionMode.EXTERNAL + turn_detection_mode: TurnDetectionMode = TurnDetectionMode.FIXED # Output formatting speaker_active_format: str | None = None @@ -230,6 +236,7 @@ class SpeechmaticsSTTService(STTService): end_of_utterance_max_delay: float | None = None punctuation_overrides: dict | None = None include_partials: bool | None = None + split_sentences: bool | None = None # Diarization enable_diarization: bool | None = None @@ -326,7 +333,10 @@ class SpeechmaticsSTTService(STTService): ) # Framework options - self._enable_vad: bool = self._config.end_of_utterance_mode != EndOfUtteranceMode.EXTERNAL + self._enable_vad: bool = self._config.end_of_utterance_mode not in [ + EndOfUtteranceMode.FIXED, + EndOfUtteranceMode.EXTERNAL, + ] self._speaker_active_format: str = params.speaker_active_format self._speaker_passive_format: str = ( params.speaker_passive_format or params.speaker_active_format @@ -487,6 +497,7 @@ class SpeechmaticsSTTService(STTService): "end_of_utterance_max_delay", "punctuation_overrides", "include_partials", + "split_sentences", "enable_diarization", "speaker_sensitivity", "max_speakers", @@ -501,6 +512,11 @@ class SpeechmaticsSTTService(STTService): if hasattr(config, key): setattr(config, key, value) + # Enable sentences + config.speech_segment_config = SpeechSegmentConfig( + emit_sentences=params.split_sentences or False + ) + # Return the complete config return config From d5d215668970298b6eefe2f6af8d0e75390f31f2 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 31 Dec 2025 19:07:11 +0000 Subject: [PATCH 5/8] Updated changelog. --- changelog/3328.fixed.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/changelog/3328.fixed.md b/changelog/3328.fixed.md index 695ca0045..6f09c2386 100644 --- a/changelog/3328.fixed.md +++ b/changelog/3328.fixed.md @@ -1,3 +1,4 @@ -- Fix to `SpeechmaticsSTTService` for version `0.0.99+`: - - Changed from using `User[Started/Stopped]SpeakingFrame` to `VADUser[Started/Stopped]SpeakingFrame` frames - - Only emit VAD + interruption frames if VAD is enabled within the plugin (modes other than `TurnDetectionMode.EXTERNAL`) +- Updated `SpeechmaticsSTTService` for version `0.0.99+`: + - Fixed `SpeechmaticsSTTService` to listen for `VADUserStoppedSpeakingFrame` in order to finalize transcription. + - Default to `TurnDetectionMode.FIXED` for Pipecat-controlled end of turn detection. + - Only emit VAD + interruption frames if VAD is enabled within the plugin (modes other than `TurnDetectionMode.FIXED` or `TurnDetectionMode.EXTERNAL`). From 3ec89e49bff27028c31a3d13603b8c847fbd40f5 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 7 Jan 2026 07:41:49 -0800 Subject: [PATCH 6/8] Added changelog for `split_sentences` and code tidy for end of turn handling. --- changelog/3328.added.md | 1 + src/pipecat/services/speechmatics/stt.py | 15 ++++----------- 2 files changed, 5 insertions(+), 11 deletions(-) create mode 100644 changelog/3328.added.md diff --git a/changelog/3328.added.md b/changelog/3328.added.md new file mode 100644 index 000000000..db793e828 --- /dev/null +++ b/changelog/3328.added.md @@ -0,0 +1 @@ +- Added `split_sentences` parameter to `SpeechmaticsSTTService` to control sentence splitting behavior for finals on sentence boundaries. diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 38f04cee7..2b7a783e5 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -27,6 +27,7 @@ from pipecat.frames.frames import ( InterimTranscriptionFrame, StartFrame, TranscriptionFrame, + UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, ) @@ -621,12 +622,8 @@ class SpeechmaticsSTTService(STTService): """ logger.debug(f"{self} StartOfTurn received") # await self.start_processing_metrics() - - # Emit VAD events if enabled - if self._enable_vad: - await self.push_interruption_task_frame_and_wait() - logger.debug(f"{self} sending VADUserStartedSpeakingFrame") - await self.broadcast_frame(VADUserStartedSpeakingFrame) + await self.push_interruption_task_frame_and_wait() + await self.broadcast_frame(VADUserStartedSpeakingFrame) async def _handle_end_of_turn(self, message: dict[str, Any]) -> None: """Handle EndOfTurn events. @@ -645,11 +642,7 @@ class SpeechmaticsSTTService(STTService): """ logger.debug(f"{self} EndOfTurn received") # await self.stop_processing_metrics() - - # Emit VAD events if enabled - if self._enable_vad: - logger.debug(f"{self} sending VADUserStoppedSpeakingFrame") - await self.broadcast_frame(VADUserStoppedSpeakingFrame) + await self.broadcast_frame(UserStoppedSpeakingFrame) async def _handle_speakers_result(self, message: dict[str, Any]) -> None: """Handle SpeakersResult events. From ecfd93544ab18f633f6c0f5a0ff4dc6d48d53f56 Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 7 Jan 2026 07:43:47 -0800 Subject: [PATCH 7/8] Correction to `UserStartedSpeakingFrame` timing. --- src/pipecat/services/speechmatics/stt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 2b7a783e5..4cfc5291b 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -27,6 +27,7 @@ from pipecat.frames.frames import ( InterimTranscriptionFrame, StartFrame, TranscriptionFrame, + UserStartedSpeakingFrame, UserStoppedSpeakingFrame, VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, @@ -622,8 +623,8 @@ class SpeechmaticsSTTService(STTService): """ logger.debug(f"{self} StartOfTurn received") # await self.start_processing_metrics() + await self.broadcast_frame(UserStartedSpeakingFrame) await self.push_interruption_task_frame_and_wait() - await self.broadcast_frame(VADUserStartedSpeakingFrame) async def _handle_end_of_turn(self, message: dict[str, Any]) -> None: """Handle EndOfTurn events. From 3e00a16f0f7a16263e796f41d6f3319ca6dee7aa Mon Sep 17 00:00:00 2001 From: Sam Sykes Date: Wed, 7 Jan 2026 07:45:26 -0800 Subject: [PATCH 8/8] Remove unused import and correction to docs. --- src/pipecat/services/speechmatics/stt.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/pipecat/services/speechmatics/stt.py b/src/pipecat/services/speechmatics/stt.py index 4cfc5291b..651dbfea4 100644 --- a/src/pipecat/services/speechmatics/stt.py +++ b/src/pipecat/services/speechmatics/stt.py @@ -29,7 +29,6 @@ from pipecat.frames.frames import ( TranscriptionFrame, UserStartedSpeakingFrame, UserStoppedSpeakingFrame, - VADUserStartedSpeakingFrame, VADUserStoppedSpeakingFrame, ) from pipecat.processors.frame_processor import FrameDirection @@ -615,7 +614,7 @@ class SpeechmaticsSTTService(STTService): The service will: - Send a BotInterruptionFrame upstream to stop bot speech - - Send a VADUserStartedSpeakingFrame downstream to notify other components + - Send a UserStartedSpeakingFrame downstream to notify other components - Start metrics collection for measuring response times Args: @@ -636,7 +635,7 @@ class SpeechmaticsSTTService(STTService): The service will: - Stop processing metrics collection - - Send a VADUserStoppedSpeakingFrame to signal turn completion + - Send a UserStoppedSpeakingFrame to signal turn completion Args: message: the message payload.