turn on/off openai vad

examples/foundational/19-openai-realtime-beta.py

@@ -20,13 +20,12 @@ from pipecat.processors.aggregators.openai_llm_context import (
     OpenAILLMContext,
 )
 from pipecat.services.openai_realtime_beta import (
-    InputAudioTranscription,
     OpenAILLMServiceRealtimeBeta,
     SessionProperties,
-    TurnDetection,
 )
 from pipecat.transports.services.daily import DailyParams, DailyTransport
 from pipecat.vad.silero import SileroVADAnalyzer
+from pipecat.vad.vad_analyzer import VADParams
 
 load_dotenv(override=True)
 
@@ -76,15 +75,19 @@ async def main():
                 audio_out_enabled=True,
                 audio_out_sample_rate=24000,
                 transcription_enabled=False,
-                vad_enabled=False,
-                vad_analyzer=SileroVADAnalyzer(),
+                vad_enabled=True,
+                vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)),
                 vad_audio_passthrough=True,
             ),
         )
 
         session_properties = SessionProperties(
-            input_audio_transcription=InputAudioTranscription(),
-            turn_detection=TurnDetection(silence_duration_ms=1000),
+            # input_audio_transcription=InputAudioTranscription(),
+            # Set openai TurnDetection parameters. Not setting this at all will turn it
+            # on by default
+            # turn_detection=TurnDetection(silence_duration_ms=1000),
+            # Or set to False to disable openai turn detection and use transport VAD
+            turn_detection=False,
             tools=tools,
             instructions="""
 Your knowledge cutoff is 2023-10. You are a helpful and friendly AI.

src/pipecat/services/openai_realtime_beta/events.py

@@ -2,6 +2,7 @@ import json
 import uuid
 from typing import Any, Dict, List, Literal, Optional, Union
 
+from loguru import logger
 from pydantic import BaseModel, Field
 
 #
@@ -27,7 +28,8 @@ class SessionProperties(BaseModel):
     input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None
     output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None
     input_audio_transcription: Optional[InputAudioTranscription] = None
-    turn_detection: Optional[TurnDetection] = None
+    # set turn_detection to False to disable turn detection
+    turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None)
     tools: Optional[List[Dict]] = None
     tool_choice: Optional[Literal["auto", "none", "required"]] = None
     temperature: Optional[float] = None
@@ -100,6 +102,17 @@ class SessionUpdateEvent(ClientEvent):
     type: Literal["session.update"] = "session.update"
     session: SessionProperties
 
+    def model_dump(self, *args, **kwargs) -> Dict[str, Any]:
+        logger.debug(f"!!! SessionUpdateEvent.model_dump: {self}")
+        dump = super().model_dump(*args, **kwargs)
+
+        # Handle turn_detection so that False is serialized as null
+        if "turn_detection" in dump["session"]:
+            if dump["session"]["turn_detection"] is False:
+                dump["session"]["turn_detection"] = None
+
+        return dump
+
 
 class InputAudioBufferAppendEvent(ClientEvent):
     type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append"

src/pipecat/services/openai_realtime_beta/llm_and_context.py

@@ -123,7 +123,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService):
         session_properties: events.SessionProperties = events.SessionProperties(),
         start_audio_paused: bool = False,
         send_transcription_frames: bool = True,
-        send_user_started_speaking_frames: bool = True,
+        send_user_started_speaking_frames: bool = False,
         **kwargs,
     ):
         super().__init__(base_url=base_url, **kwargs)
@@ -133,6 +133,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService):
         self._session_properties = session_properties
         self._audio_input_paused = start_audio_paused
         self._send_transcription_frames = send_transcription_frames
+        # todo: wire _send_user_started_speaking_frames up correctly
         self._send_user_started_speaking_frames = send_user_started_speaking_frames
         self._websocket = None
         self._receive_task = None
@@ -158,7 +159,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService):
         await self._disconnect()
 
     async def send_client_event(self, event: events.ClientEvent):
-        await self._ws_send(event.dict(exclude_none=True))
+        await self._ws_send(event.model_dump(exclude_none=True))
 
     async def _ws_send(self, realtime_message):
         try:
@@ -359,9 +360,6 @@ class OpenAILLMServiceRealtimeBeta(LLMService):
         context = self._context
         messages = context.get_unsent_messages()
         context.update_all_messages_sent()
-        logger.debug(
-            f"Sending message context updates: {context._marker} {context.get_messages_for_logging()}"
-        )
 
         items = []
         for m in messages:
@@ -401,10 +399,20 @@ class OpenAILLMServiceRealtimeBeta(LLMService):
         await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload))
 
     async def _handle_interruption(self, frame):
+        await self.send_client_event(events.InputAudioBufferClearEvent())
+        await self.send_client_event(events.ResponseCancelEvent())
         await self.stop_all_metrics()
         await self.push_frame(LLMFullResponseEndFrame())
         await self.push_frame(TTSStoppedFrame())
-        # todo: track whether a response is in progress and cancel it with a response.cancela nd input_audio_buffer.clear (?)
+
+    async def _handle_user_started_speaking(self, frame):
+        pass
+
+    async def _handle_user_stopped_speaking(self, frame):
+        if self._session_properties.turn_detection is None:
+            await self.send_client_event(events.InputAudioBufferCommitEvent())
+            await self.send_client_event(events.ResponseCreateEvent())
+        pass
 
     async def process_frame(self, frame: Frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
@@ -422,6 +430,10 @@ class OpenAILLMServiceRealtimeBeta(LLMService):
                 await self._send_user_audio(frame)
         elif isinstance(frame, StartInterruptionFrame):
             await self._handle_interruption(frame)
+        elif isinstance(frame, UserStartedSpeakingFrame):
+            await self._handle_user_started_speaking(frame)
+        elif isinstance(frame, UserStoppedSpeakingFrame):
+            await self._handle_user_stopped_speaking(frame)
         elif isinstance(frame, _InternalMessagesUpdateFrame):
             self._context = frame.context
             await self._send_messages_context_update()