diff --git a/src/pipecat/services/gladia/config.py b/src/pipecat/services/gladia/config.py index ed160a36e..5fd5bfdac 100644 --- a/src/pipecat/services/gladia/config.py +++ b/src/pipecat/services/gladia/config.py @@ -169,6 +169,9 @@ class GladiaInputParams(BaseModel): pre_processing: Audio pre-processing options realtime_processing: Real-time processing features messages_config: WebSocket message filtering options + enable_vad: Enable VAD to trigger end of utterance detection. This should be used + without any other VAD enabled in the agent and will emit the speaker started + and stopped frames. Defaults to False. """ encoding: Optional[str] = "wav/pcm" @@ -182,3 +185,4 @@ class GladiaInputParams(BaseModel): pre_processing: Optional[PreProcessingConfig] = None realtime_processing: Optional[RealtimeProcessingConfig] = None messages_config: Optional[MessagesConfig] = None + enable_vad: bool = False diff --git a/src/pipecat/services/gladia/stt.py b/src/pipecat/services/gladia/stt.py index 4ba0a2ffd..9a20d6dcb 100644 --- a/src/pipecat/services/gladia/stt.py +++ b/src/pipecat/services/gladia/stt.py @@ -28,6 +28,8 @@ from pipecat.frames.frames import ( StartFrame, TranscriptionFrame, TranslationFrame, + UserStartedSpeakingFrame, + UserStoppedSpeakingFrame, ) from pipecat.services.gladia.config import GladiaInputParams from pipecat.services.stt_service import WebsocketSTTService @@ -266,6 +268,9 @@ class GladiaSTTService(WebsocketSTTService): self._max_buffer_size = max_buffer_size self._buffer_lock = asyncio.Lock() + # VAD state tracking + self._is_speaking = False + def __str__(self): return f"{self.name} [{self._session_id}]" @@ -507,6 +512,30 @@ class GladiaSTTService(WebsocketSTTService): await self.stop_ttfb_metrics() await self.stop_processing_metrics() + async def _on_speech_started(self): + """Handle speech start event from Gladia. + + Triggers interruption and emits UserStartedSpeakingFrame when VAD is enabled. + """ + if not self._params.enable_vad or self._is_speaking: + return + logger.debug(f"{self} User started speaking") + self._is_speaking = True + # Push interruption first to stop the bot, then notify about user speaking + await self.push_interruption_task_frame_and_wait() + await self.push_frame(UserStartedSpeakingFrame()) + + async def _on_speech_ended(self): + """Handle speech end event from Gladia. + + Emits UserStoppedSpeakingFrame when VAD is enabled. + """ + if not self._params.enable_vad or not self._is_speaking: + return + self._is_speaking = False + await self.push_frame(UserStoppedSpeakingFrame()) + logger.debug(f"{self} User stopped speaking") + async def _send_audio(self, audio: bytes): """Send audio chunk with proper message format.""" if self._websocket and self._websocket.state is State.OPEN: @@ -599,6 +628,10 @@ class GladiaSTTService(WebsocketSTTService): translation, "", time_now_iso8601(), translated_language ) ) + elif content["type"] == "speech_start": + await self._on_speech_started() + elif content["type"] == "speech_end": + await self._on_speech_ended() except json.JSONDecodeError: logger.warning(f"{self} Received non-JSON message: {message}")