From 31916ed9fd19eb460eb71534e73a8d4bcfcd84a4 Mon Sep 17 00:00:00 2001 From: Kwindla Hultman Kramer Date: Mon, 7 Oct 2024 22:09:18 -0700 Subject: [PATCH] turn on/off openai vad --- .../foundational/19-openai-realtime-beta.py | 15 +++++++----- .../services/openai_realtime_beta/events.py | 15 +++++++++++- .../openai_realtime_beta/llm_and_context.py | 24 ++++++++++++++----- 3 files changed, 41 insertions(+), 13 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 16b291ef2..b687eefc1 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -20,13 +20,12 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContext, ) from pipecat.services.openai_realtime_beta import ( - InputAudioTranscription, OpenAILLMServiceRealtimeBeta, SessionProperties, - TurnDetection, ) from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer +from pipecat.vad.vad_analyzer import VADParams load_dotenv(override=True) @@ -76,15 +75,19 @@ async def main(): audio_out_enabled=True, audio_out_sample_rate=24000, transcription_enabled=False, - vad_enabled=False, - vad_analyzer=SileroVADAnalyzer(), + vad_enabled=True, + vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.8)), vad_audio_passthrough=True, ), ) session_properties = SessionProperties( - input_audio_transcription=InputAudioTranscription(), - turn_detection=TurnDetection(silence_duration_ms=1000), + # input_audio_transcription=InputAudioTranscription(), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + # turn_detection=TurnDetection(silence_duration_ms=1000), + # Or set to False to disable openai turn detection and use transport VAD + turn_detection=False, tools=tools, instructions=""" Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. diff --git a/src/pipecat/services/openai_realtime_beta/events.py b/src/pipecat/services/openai_realtime_beta/events.py index 4f308b3ce..48492e972 100644 --- a/src/pipecat/services/openai_realtime_beta/events.py +++ b/src/pipecat/services/openai_realtime_beta/events.py @@ -2,6 +2,7 @@ import json import uuid from typing import Any, Dict, List, Literal, Optional, Union +from loguru import logger from pydantic import BaseModel, Field # @@ -27,7 +28,8 @@ class SessionProperties(BaseModel): input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = None input_audio_transcription: Optional[InputAudioTranscription] = None - turn_detection: Optional[TurnDetection] = None + # set turn_detection to False to disable turn detection + turn_detection: Optional[Union[TurnDetection, bool]] = Field(default=None) tools: Optional[List[Dict]] = None tool_choice: Optional[Literal["auto", "none", "required"]] = None temperature: Optional[float] = None @@ -100,6 +102,17 @@ class SessionUpdateEvent(ClientEvent): type: Literal["session.update"] = "session.update" session: SessionProperties + def model_dump(self, *args, **kwargs) -> Dict[str, Any]: + logger.debug(f"!!! SessionUpdateEvent.model_dump: {self}") + dump = super().model_dump(*args, **kwargs) + + # Handle turn_detection so that False is serialized as null + if "turn_detection" in dump["session"]: + if dump["session"]["turn_detection"] is False: + dump["session"]["turn_detection"] = None + + return dump + class InputAudioBufferAppendEvent(ClientEvent): type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" diff --git a/src/pipecat/services/openai_realtime_beta/llm_and_context.py b/src/pipecat/services/openai_realtime_beta/llm_and_context.py index 35397d95d..f21f2066b 100644 --- a/src/pipecat/services/openai_realtime_beta/llm_and_context.py +++ b/src/pipecat/services/openai_realtime_beta/llm_and_context.py @@ -123,7 +123,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService): session_properties: events.SessionProperties = events.SessionProperties(), start_audio_paused: bool = False, send_transcription_frames: bool = True, - send_user_started_speaking_frames: bool = True, + send_user_started_speaking_frames: bool = False, **kwargs, ): super().__init__(base_url=base_url, **kwargs) @@ -133,6 +133,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService): self._session_properties = session_properties self._audio_input_paused = start_audio_paused self._send_transcription_frames = send_transcription_frames + # todo: wire _send_user_started_speaking_frames up correctly self._send_user_started_speaking_frames = send_user_started_speaking_frames self._websocket = None self._receive_task = None @@ -158,7 +159,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService): await self._disconnect() async def send_client_event(self, event: events.ClientEvent): - await self._ws_send(event.dict(exclude_none=True)) + await self._ws_send(event.model_dump(exclude_none=True)) async def _ws_send(self, realtime_message): try: @@ -359,9 +360,6 @@ class OpenAILLMServiceRealtimeBeta(LLMService): context = self._context messages = context.get_unsent_messages() context.update_all_messages_sent() - logger.debug( - f"Sending message context updates: {context._marker} {context.get_messages_for_logging()}" - ) items = [] for m in messages: @@ -401,10 +399,20 @@ class OpenAILLMServiceRealtimeBeta(LLMService): await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) async def _handle_interruption(self, frame): + await self.send_client_event(events.InputAudioBufferClearEvent()) + await self.send_client_event(events.ResponseCancelEvent()) await self.stop_all_metrics() await self.push_frame(LLMFullResponseEndFrame()) await self.push_frame(TTSStoppedFrame()) - # todo: track whether a response is in progress and cancel it with a response.cancela nd input_audio_buffer.clear (?) + + async def _handle_user_started_speaking(self, frame): + pass + + async def _handle_user_stopped_speaking(self, frame): + if self._session_properties.turn_detection is None: + await self.send_client_event(events.InputAudioBufferCommitEvent()) + await self.send_client_event(events.ResponseCreateEvent()) + pass async def process_frame(self, frame: Frame, direction: FrameDirection): await super().process_frame(frame, direction) @@ -422,6 +430,10 @@ class OpenAILLMServiceRealtimeBeta(LLMService): await self._send_user_audio(frame) elif isinstance(frame, StartInterruptionFrame): await self._handle_interruption(frame) + elif isinstance(frame, UserStartedSpeakingFrame): + await self._handle_user_started_speaking(frame) + elif isinstance(frame, UserStoppedSpeakingFrame): + await self._handle_user_stopped_speaking(frame) elif isinstance(frame, _InternalMessagesUpdateFrame): self._context = frame.context await self._send_messages_context_update()