diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index 47efb5887..3a5af542e 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -17,8 +17,8 @@ from pipecat.processors.aggregators.openai_llm_context import ( ) from pipecat.services.openai_realtime_beta import ( OpenAILLMServiceRealtimeBeta, - OpenAITurnDetection, - RealtimeSessionProperties, + TurnDetection, + SessionProperties, ) from pipecat.transports.services.daily import DailyParams, DailyTransport from pipecat.vad.silero import SileroVADAnalyzer @@ -83,8 +83,8 @@ async def main(): ), ) - session_properties = RealtimeSessionProperties( - turn_detection=OpenAITurnDetection(silence_duration_ms=1000), + session_properties = SessionProperties( + turn_detection=TurnDetection(silence_duration_ms=1000), tools=tools, instructions=""" Your knowledge cutoff is 2023-10. You are a helpful and friendly AI. diff --git a/src/pipecat/services/openai_realtime_beta/__init__.py b/src/pipecat/services/openai_realtime_beta/__init__.py new file mode 100644 index 000000000..1df8c5f44 --- /dev/null +++ b/src/pipecat/services/openai_realtime_beta/__init__.py @@ -0,0 +1,2 @@ +from .llm_and_context import OpenAILLMServiceRealtimeBeta +from .client_events import SessionProperties, TurnDetection diff --git a/src/pipecat/services/openai_realtime_beta/client_events.py b/src/pipecat/services/openai_realtime_beta/client_events.py new file mode 100644 index 000000000..578e56822 --- /dev/null +++ b/src/pipecat/services/openai_realtime_beta/client_events.py @@ -0,0 +1,33 @@ +from pydantic import BaseModel +from typing import Dict, List, Optional, Literal + + +class InputAudioTranscription(BaseModel): + model: Optional[str] = "whisper-1" + + +class TurnDetection(BaseModel): + type: Optional[Literal["server_vad"]] = "server_vad" + threshold: Optional[float] = 0.5 + prefix_padding_ms: Optional[int] = 300 + silence_duration_ms: Optional[int] = 800 + + +class SessionProperties(BaseModel): + modalities: Optional[List[Literal["text", "audio"]]] = ["text", "audio"] + instructions: Optional[str] = None + voice: Optional[str] = "alloy" + input_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = "pcm16" + output_audio_format: Optional[Literal["pcm16", "g711_ulaw", "g711_alaw"]] = "pcm16" + input_audio_transcription: Optional[InputAudioTranscription] = InputAudioTranscription() + turn_detection: Optional[TurnDetection] = TurnDetection() + tools: Optional[List[Dict]] = [] + tool_choice: Optional[Literal["auto", "none", "required"]] = "auto" + temperature: Optional[float] = 0.8 + max_response_output_tokens: Optional[int] = 4096 + + +class SessionUpdateEvent(BaseModel): + event_id: str + type: Literal["session.update"] + session: SessionProperties diff --git a/src/pipecat/services/openai_realtime_beta.py b/src/pipecat/services/openai_realtime_beta/llm_and_context.py similarity index 94% rename from src/pipecat/services/openai_realtime_beta.py rename to src/pipecat/services/openai_realtime_beta/llm_and_context.py index 819354ac8..5781ca00a 100644 --- a/src/pipecat/services/openai_realtime_beta.py +++ b/src/pipecat/services/openai_realtime_beta/llm_and_context.py @@ -1,11 +1,11 @@ import asyncio import base64 import random +import traceback import json import websockets from copy import deepcopy -from typing import List, Optional from pydantic import BaseModel, Field @@ -37,6 +37,8 @@ from pipecat.processors.aggregators.openai_llm_context import ( OpenAILLMContextFrame, ) +from . import client_events as events + from loguru import logger # temp: websocket logger @@ -112,29 +114,13 @@ class OpenAITurnDetection(BaseModel): ) -class RealtimeSessionProperties(BaseModel): - modalities: List[str] = Field(default=["text", "audio"]) - instructions: str = Field(default="") - voice: str = Field(default="alloy") - input_audio_format: str = Field(default="pcm16") - output_audio_format: str = Field(default="pcm16") - input_audio_transcription: Optional[OpenAIInputTranscription] = Field( - default=OpenAIInputTranscription() - ) - turn_detection: Optional[OpenAITurnDetection] = Field(default=None) - tools: List[dict] = Field(default=[]) - tool_choice: str = Field(default="auto") - temperature: float = Field(default=0.8) - max_response_output_tokens: int = Field(default=4096) - - class OpenAILLMServiceRealtimeBeta(LLMService): def __init__( self, *, api_key: str, base_url="wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01", - session_properties: RealtimeSessionProperties = RealtimeSessionProperties(), + session_properties: events.SessionProperties = events.SessionProperties(), **kwargs, ): super().__init__(base_url=base_url, **kwargs) @@ -175,7 +161,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService): await self._ws_send( { "type": "session.update", - "session": self._session_properties.dict(), + "session": self._session_properties.dict(exclude_none=True), } ) @@ -223,6 +209,8 @@ class OpenAILLMServiceRealtimeBeta(LLMService): if not msg: continue if msg["type"] == "session.created": + # session.created is received right after connecting. send a message + # to configure the session properties. await self.update_session_properties() elif msg["type"] == "session.updated": self._session_properties = msg["session"] @@ -326,7 +314,7 @@ class OpenAILLMServiceRealtimeBeta(LLMService): except asyncio.CancelledError: pass except Exception as e: - logger.error(f"{self} exception: {e}") + logger.error(f"{self} exception: {e}\n\nStack trace:\n{traceback.format_exc()}") async def _handle_function_call_items(self, items): total_items = len(items)