diff --git a/CHANGELOG.md b/CHANGELOG.md index 9493c9d06..7079f21f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Deprecated +- `pipecat.service.openai_realtime` is now deprecated, use + `pipecat.services.openai.realtime` instead or + `pipecat.services.azure.realtime` for Azure Realtime. + - `pipecat.service.aws_nova_sonic` is now deprecated, use `pipecat.services.aws.nova_sonic` instead. diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 0f9309f3d..f182d7c8c 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -24,14 +24,15 @@ from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioNoiseReduction, InputAudioTranscription, - OpenAIRealtimeLLMService, SemanticTurnDetection, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index a39826b81..c4b0fc02a 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -21,13 +21,14 @@ from pipecat.pipeline.task import PipelineParams, PipelineTask from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport +from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( - AzureRealtimeLLMService, +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioTranscription, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py index b5d1d73e1..bb63a4814 100644 --- a/examples/foundational/19b-openai-realtime-text.py +++ b/examples/foundational/19b-openai-realtime-text.py @@ -22,16 +22,17 @@ from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext from pipecat.processors.transcript_processor import TranscriptProcessor from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport -from pipecat.services.cartesia import CartesiaTTSService +from pipecat.services.cartesia.tts import CartesiaTTSService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioNoiseReduction, InputAudioTranscription, - OpenAIRealtimeLLMService, SemanticTurnDetection, SessionProperties, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 52c7d8f49..629a17c67 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -25,13 +25,14 @@ from pipecat.runner.types import RunnerArguments from pipecat.runner.utils import create_transport from pipecat.services.deepgram.stt import DeepgramSTTService from pipecat.services.llm_service import FunctionCallParams -from pipecat.services.openai_realtime import ( +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, InputAudioTranscription, - OpenAIRealtimeLLMService, SessionProperties, TurnDetection, ) -from pipecat.services.openai_realtime.events import AudioConfiguration, AudioInput +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService from pipecat.transports.base_transport import BaseTransport, TransportParams from pipecat.transports.daily.transport import DailyParams from pipecat.transports.websocket.fastapi import FastAPIWebsocketParams diff --git a/src/pipecat/services/ai_service.py b/src/pipecat/services/ai_service.py index eebdbfb0b..759efcf00 100644 --- a/src/pipecat/services/ai_service.py +++ b/src/pipecat/services/ai_service.py @@ -97,9 +97,7 @@ class AIService(FrameProcessor): pass async def _update_settings(self, settings: Mapping[str, Any]): - from pipecat.services.openai_realtime_beta.events import ( - SessionProperties, - ) + from pipecat.services.openai.realtime.events import SessionProperties for key, value in settings.items(): logger.debug("Update request for:", key, value) @@ -111,9 +109,7 @@ class AIService(FrameProcessor): logger.debug("Attempting to update", key, value) try: - from pipecat.services.openai_realtime_beta.events import ( - TurnDetection, - ) + from pipecat.services.openai.realtime.events import TurnDetection if isinstance(self._session_properties, SessionProperties): current_properties = self._session_properties diff --git a/src/pipecat/services/azure/realtime/__init__.py b/src/pipecat/services/azure/realtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/azure/realtime/llm.py b/src/pipecat/services/azure/realtime/llm.py new file mode 100644 index 000000000..fb420e10b --- /dev/null +++ b/src/pipecat/services/azure/realtime/llm.py @@ -0,0 +1,65 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Azure OpenAI Realtime LLM service implementation.""" + +from loguru import logger + +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService + +try: + from websockets.asyncio.client import connect as websocket_connect +except ModuleNotFoundError as e: + logger.error(f"Exception: {e}") + logger.error("In order to use Azure Realtime, you need to `pip install pipecat-ai[openai]`.") + raise Exception(f"Missing module: {e}") + + +class AzureRealtimeLLMService(OpenAIRealtimeLLMService): + """Azure OpenAI Realtime LLM service with Azure-specific authentication. + + Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, + using Azure's authentication headers and endpoint format. Provides the same + real-time audio and text communication capabilities as the base OpenAI service. + """ + + def __init__( + self, + *, + api_key: str, + base_url: str, + **kwargs, + ): + """Initialize Azure Realtime LLM service. + + Args: + api_key: The API key for the Azure OpenAI service. + base_url: The full Azure WebSocket endpoint URL including api-version and deployment. + Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" + **kwargs: Additional arguments passed to parent OpenAIRealtimeLLMService. + """ + super().__init__(base_url=base_url, api_key=api_key, **kwargs) + self.api_key = api_key + self.base_url = base_url + + async def _connect(self): + try: + if self._websocket: + # Here we assume that if we have a websocket, we are connected. We + # handle disconnections in the send/recv code paths. + return + + logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") + self._websocket = await websocket_connect( + uri=self.base_url, + additional_headers={ + "api-key": self.api_key, + }, + ) + self._receive_task = self.create_task(self._receive_task_handler()) + except Exception as e: + logger.error(f"{self} initialization error: {e}") + self._websocket = None diff --git a/src/pipecat/services/openai/__init__.py b/src/pipecat/services/openai/__init__.py index 4decac126..d913c5ee0 100644 --- a/src/pipecat/services/openai/__init__.py +++ b/src/pipecat/services/openai/__init__.py @@ -10,6 +10,7 @@ from pipecat.services import DeprecatedModuleProxy from .image import * from .llm import * +from .realtime import * from .stt import * from .tts import * diff --git a/src/pipecat/services/openai/realtime/__init__.py b/src/pipecat/services/openai/realtime/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/pipecat/services/openai/realtime/context.py b/src/pipecat/services/openai/realtime/context.py new file mode 100644 index 000000000..cb1c0a9f5 --- /dev/null +++ b/src/pipecat/services/openai/realtime/context.py @@ -0,0 +1,272 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""OpenAI Realtime LLM context and aggregator implementations.""" + +import copy +import json + +from loguru import logger + +from pipecat.frames.frames import ( + Frame, + FunctionCallResultFrame, + InterimTranscriptionFrame, + LLMMessagesUpdateFrame, + LLMSetToolsFrame, + LLMTextFrame, + TranscriptionFrame, +) +from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext +from pipecat.processors.frame_processor import FrameDirection +from pipecat.services.openai.llm import ( + OpenAIAssistantContextAggregator, + OpenAIUserContextAggregator, +) + +from . import events +from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame + + +class OpenAIRealtimeLLMContext(OpenAILLMContext): + """OpenAI Realtime LLM context with session management and message conversion. + + Extends the standard OpenAI LLM context to support real-time session properties, + instruction management, and conversion between standard message formats and + realtime conversation items. + """ + + def __init__(self, messages=None, tools=None, **kwargs): + """Initialize the OpenAIRealtimeLLMContext. + + Args: + messages: Initial conversation messages. Defaults to None. + tools: Available function tools. Defaults to None. + **kwargs: Additional arguments passed to parent OpenAILLMContext. + """ + super().__init__(messages=messages, tools=tools, **kwargs) + self.__setup_local() + + def __setup_local(self): + self.llm_needs_settings_update = True + self.llm_needs_initial_messages = True + self._session_instructions = "" + + return + + @staticmethod + def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": + """Upgrade a standard OpenAI LLM context to a realtime context. + + Args: + obj: The OpenAILLMContext instance to upgrade. + + Returns: + The upgraded OpenAIRealtimeLLMContext instance. + """ + if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): + obj.__class__ = OpenAIRealtimeLLMContext + obj.__setup_local() + return obj + + # todo + # - finish implementing all frames + + def from_standard_message(self, message): + """Convert a standard message format to a realtime conversation item. + + Args: + message: The standard message dictionary to convert. + + Returns: + A ConversationItem instance for the realtime API. + """ + if message.get("role") == "user": + content = message.get("content") + if isinstance(message.get("content"), list): + content = "" + for c in message.get("content"): + if c.get("type") == "text": + content += " " + c.get("text") + else: + logger.error( + f"Unhandled content type in context message: {c.get('type')} - {message}" + ) + return events.ConversationItem( + role="user", + type="message", + content=[events.ItemContent(type="input_text", text=content)], + ) + if message.get("role") == "assistant" and message.get("tool_calls"): + tc = message.get("tool_calls")[0] + return events.ConversationItem( + type="function_call", + call_id=tc["id"], + name=tc["function"]["name"], + arguments=tc["function"]["arguments"], + ) + logger.error(f"Unhandled message type in from_standard_message: {message}") + + def get_messages_for_initializing_history(self): + """Get conversation items for initializing the realtime session history. + + Converts the context's messages to a format suitable for the realtime API, + handling system instructions and conversation history packaging. + + Returns: + List of conversation items for session initialization. + """ + # We can't load a long conversation history into the openai realtime api yet. (The API/model + # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So + # our general strategy until this is fixed is just to put everything into a first "user" + # message as a single input. + if not self.messages: + return [] + + messages = copy.deepcopy(self.messages) + + # If we have a "system" message as our first message, let's pull that out into session + # "instructions" + if messages[0].get("role") == "system": + self.llm_needs_settings_update = True + system = messages.pop(0) + content = system.get("content") + if isinstance(content, str): + self._session_instructions = content + elif isinstance(content, list): + self._session_instructions = content[0].get("text") + if not messages: + return [] + + # If we have just a single "user" item, we can just send it normally + if len(messages) == 1 and messages[0].get("role") == "user": + return [self.from_standard_message(messages[0])] + + # Otherwise, let's pack everything into a single "user" message with a bit of + # explanation for the LLM + intro_text = """ + This is a previously saved conversation. Please treat this conversation history as a + starting point for the current conversation.""" + + trailing_text = """ + This is the end of the previously saved conversation. Please continue the conversation + from here. If the last message is a user instruction or question, act on that instruction + or answer the question. If the last message is an assistant response, simple say that you + are ready to continue the conversation.""" + + return [ + { + "role": "user", + "type": "message", + "content": [ + { + "type": "input_text", + "text": "\n\n".join( + [intro_text, json.dumps(messages, indent=2), trailing_text] + ), + } + ], + } + ] + + def add_user_content_item_as_message(self, item): + """Add a user content item as a standard message to the context. + + Args: + item: The conversation item to add as a user message. + """ + message = { + "role": "user", + "content": [{"type": "text", "text": item.content[0].transcript}], + } + self.add_message(message) + + +class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): + """User context aggregator for OpenAI Realtime API. + + Handles user input frames and generates appropriate context updates + for the realtime conversation, including message updates and tool settings. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + + async def process_frame( + self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM + ): + """Process incoming frames and handle realtime-specific frame types. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + await super().process_frame(frame, direction) + # Parent does not push LLMMessagesUpdateFrame. This ensures that in a typical pipeline, + # messages are only processed by the user context aggregator, which is generally what we want. But + # we also need to send new messages over the websocket, so the openai realtime API has them + # in its context. + if isinstance(frame, LLMMessagesUpdateFrame): + await self.push_frame(RealtimeMessagesUpdateFrame(context=self._context)) + + # Parent also doesn't push the LLMSetToolsFrame. + if isinstance(frame, LLMSetToolsFrame): + await self.push_frame(frame, direction) + + async def push_aggregation(self): + """Push user input aggregation. + + Currently ignores all user input coming into the pipeline as realtime + audio input is handled directly by the service. + """ + # for the moment, ignore all user input coming into the pipeline. + # todo: think about whether/how to fix this to allow for text input from + # upstream (transport/transcription, or other sources) + pass + + +class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): + """Assistant context aggregator for OpenAI Realtime API. + + Handles assistant output frames from the realtime service, filtering + out duplicate text frames and managing function call results. + + Args: + context: The OpenAI realtime LLM context. + **kwargs: Additional arguments passed to parent aggregator. + """ + + # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, + # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We + # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames + # are process. This ensures that the context gets only one set of messages. + # OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames, + # so we need to ignore pushing those as well, as they're also TextFrames. + async def process_frame(self, frame: Frame, direction: FrameDirection): + """Process assistant frames, filtering out duplicate text content. + + Args: + frame: The frame to process. + direction: The direction of frame flow in the pipeline. + """ + if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)): + await super().process_frame(frame, direction) + + async def handle_function_call_result(self, frame: FunctionCallResultFrame): + """Handle function call result and notify the realtime service. + + Args: + frame: The function call result frame to handle. + """ + await super().handle_function_call_result(frame) + + # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, + # so we didn't have a chance to add the result to the openai realtime api context. Let's push a + # special frame to do that. + await self.push_frame( + RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM + ) diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py new file mode 100644 index 000000000..200e59d68 --- /dev/null +++ b/src/pipecat/services/openai/realtime/events.py @@ -0,0 +1,1106 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Event models and data structures for OpenAI Realtime API communication.""" + +import json +import uuid +from typing import Any, Dict, List, Literal, Optional, Union + +from pydantic import BaseModel, ConfigDict, Field + +# +# session properties +# + + +class AudioFormat(BaseModel): + """Base class for audio format configuration.""" + + type: str + + +class PCMAudioFormat(AudioFormat): + """PCM audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcm". + rate: Sample rate, always 24000 for PCM. + """ + + type: Literal["audio/pcm"] = "audio/pcm" + rate: Literal[24000] = 24000 + + +class PCMUAudioFormat(AudioFormat): + """PCMU (G.711 μ-law) audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcmu". + """ + + type: Literal["audio/pcmu"] = "audio/pcmu" + + +class PCMAAudioFormat(AudioFormat): + """PCMA (G.711 A-law) audio format configuration. + + Parameters: + type: Audio format type, always "audio/pcma". + """ + + type: Literal["audio/pcma"] = "audio/pcma" + + +class InputAudioTranscription(BaseModel): + """Configuration for audio transcription settings.""" + + model: str = "gpt-4o-transcribe" + language: Optional[str] + prompt: Optional[str] + + def __init__( + self, + model: Optional[str] = "gpt-4o-transcribe", + language: Optional[str] = None, + prompt: Optional[str] = None, + ): + """Initialize InputAudioTranscription. + + Args: + model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). + language: Optional language code for transcription. + prompt: Optional transcription hint text. + """ + super().__init__(model=model, language=language, prompt=prompt) + + +class TurnDetection(BaseModel): + """Server-side voice activity detection configuration. + + Parameters: + type: Detection type, must be "server_vad". + threshold: Voice activity detection threshold (0.0-1.0). Defaults to 0.5. + prefix_padding_ms: Padding before speech starts in milliseconds. Defaults to 300. + silence_duration_ms: Silence duration to detect speech end in milliseconds. Defaults to 500. + """ + + type: Optional[Literal["server_vad"]] = "server_vad" + threshold: Optional[float] = 0.5 + prefix_padding_ms: Optional[int] = 300 + silence_duration_ms: Optional[int] = 500 + + +class SemanticTurnDetection(BaseModel): + """Semantic-based turn detection configuration. + + Parameters: + type: Detection type, must be "semantic_vad". + eagerness: Turn detection eagerness level. Can be "low", "medium", "high", or "auto". + create_response: Whether to automatically create responses on turn detection. + interrupt_response: Whether to interrupt ongoing responses on turn detection. + """ + + type: Optional[Literal["semantic_vad"]] = "semantic_vad" + eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None + create_response: Optional[bool] = None + interrupt_response: Optional[bool] = None + + +class InputAudioNoiseReduction(BaseModel): + """Input audio noise reduction configuration. + + Parameters: + type: Noise reduction type for different microphone scenarios. + """ + + type: Optional[Literal["near_field", "far_field"]] + + +class AudioInput(BaseModel): + """Audio input configuration. + + Parameters: + format: The format of the input audio. + transcription: Configuration for input audio transcription. + noise_reduction: Configuration for input audio noise reduction. + turn_detection: Configuration for turn detection, or False to disable. + """ + + format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None + transcription: Optional[InputAudioTranscription] = None + noise_reduction: Optional[InputAudioNoiseReduction] = None + turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = None + + +class AudioOutput(BaseModel): + """Audio output configuration. + + Parameters: + format: The format of the output audio. + voice: The voice the model uses to respond. + speed: The speed of the model's spoken response. + """ + + format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None + voice: Optional[str] = None + speed: Optional[float] = None + + +class AudioConfiguration(BaseModel): + """Audio configuration for input and output. + + Parameters: + input: Configuration for input audio. + output: Configuration for output audio. + """ + + input: Optional[AudioInput] = None + output: Optional[AudioOutput] = None + + +class SessionProperties(BaseModel): + """Configuration properties for an OpenAI Realtime session. + + Parameters: + type: The type of session, always "realtime". + object: Object type identifier, always "realtime.session". + id: Unique identifier for the session. + model: The Realtime model used for this session. + output_modalities: The set of modalities the model can respond with. + instructions: System instructions for the assistant. + audio: Configuration for input and output audio. + tools: Available function tools for the assistant. + tool_choice: Tool usage strategy ("auto", "none", or "required"). + max_output_tokens: Maximum tokens in response or "inf" for unlimited. + tracing: Configuration options for tracing. + prompt: Reference to a prompt template and its variables. + expires_at: Session expiration timestamp. + include: Additional fields to include in server outputs. + """ + + type: Optional[Literal["realtime"]] = "realtime" + object: Optional[Literal["realtime.session"]] = None + id: Optional[str] = None + model: Optional[str] = None + output_modalities: Optional[List[Literal["text", "audio"]]] = None + instructions: Optional[str] = None + audio: Optional[AudioConfiguration] = None + tools: Optional[List[Dict]] = None + tool_choice: Optional[Literal["auto", "none", "required"]] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + tracing: Optional[Union[Literal["auto"], Dict]] = None + prompt: Optional[Dict] = None + expires_at: Optional[int] = None + include: Optional[List[str]] = None + + +# +# context +# + + +class ItemContent(BaseModel): + """Content within a conversation item. + + Parameters: + type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). + text: Text content for text-based items. + audio: Base64-encoded audio data for audio items. + transcript: Transcribed text for audio items. + """ + + type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] + text: Optional[str] = None + audio: Optional[str] = None # base64-encoded audio + transcript: Optional[str] = None + + +class ConversationItem(BaseModel): + """A conversation item in the realtime session. + + Parameters: + id: Unique identifier for the item, auto-generated if not provided. + object: Object type identifier for the realtime API. + type: Item type (message, function_call, or function_call_output). + status: Current status of the item. + role: Speaker role for message items (user, assistant, or system). + content: Content list for message items. + call_id: Function call identifier for function_call items. + name: Function name for function_call items. + arguments: Function arguments as JSON string for function_call items. + output: Function output as JSON string for function_call_output items. + """ + + id: str = Field(default_factory=lambda: str(uuid.uuid4().hex)) + object: Optional[Literal["realtime.item"]] = None + type: Literal["message", "function_call", "function_call_output"] + status: Optional[Literal["completed", "in_progress", "incomplete"]] = None + # role and content are present for message items + role: Optional[Literal["user", "assistant", "system"]] = None + content: Optional[List[ItemContent]] = None + # these four fields are present for function_call items + call_id: Optional[str] = None + name: Optional[str] = None + arguments: Optional[str] = None + output: Optional[str] = None + + +class RealtimeConversation(BaseModel): + """A realtime conversation session. + + Parameters: + id: Unique identifier for the conversation. + object: Object type identifier, always "realtime.conversation". + """ + + id: str + object: Literal["realtime.conversation"] + + +class ResponseProperties(BaseModel): + """Properties for configuring assistant responses. + + Parameters: + output_modalities: Output modalities for the response. Must be either ["text"] or ["audio"]. Defaults to ["audio"]. + instructions: Specific instructions for this response. + audio: Audio configuration for this response. + tools: Available tools for this response. + tool_choice: Tool usage strategy for this response. + temperature: Sampling temperature for this response. + max_output_tokens: Maximum tokens for this response. + """ + + output_modalities: Optional[List[Literal["text", "audio"]]] = ["audio"] + instructions: Optional[str] = None + audio: Optional[AudioConfiguration] = None + tools: Optional[List[Dict]] = None + tool_choice: Optional[Literal["auto", "none", "required"]] = None + temperature: Optional[float] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + + +# +# error class +# +class RealtimeError(BaseModel): + """Error information from the realtime API. + + Parameters: + type: Error type identifier. + code: Specific error code. + message: Human-readable error message. + param: Parameter name that caused the error, if applicable. + event_id: Event ID associated with the error, if applicable. + """ + + type: str + code: Optional[str] = "" + message: str + param: Optional[str] = None + event_id: Optional[str] = None + + +# +# client events +# + + +class ClientEvent(BaseModel): + """Base class for client events sent to the realtime API. + + Parameters: + event_id: Unique identifier for the event, auto-generated if not provided. + """ + + event_id: str = Field(default_factory=lambda: str(uuid.uuid4())) + + +class SessionUpdateEvent(ClientEvent): + """Event to update session properties. + + Parameters: + type: Event type, always "session.update". + session: Updated session properties. + """ + + type: Literal["session.update"] = "session.update" + session: SessionProperties + + def model_dump(self, *args, **kwargs) -> Dict[str, Any]: + """Serialize the event to a dictionary. + + Handles special serialization for turn_detection where False becomes null. + + Args: + *args: Positional arguments passed to parent model_dump. + **kwargs: Keyword arguments passed to parent model_dump. + + Returns: + Dictionary representation of the event. + """ + dump = super().model_dump(*args, **kwargs) + + # Handle turn_detection in audio.input so that False becomes null + if "audio" in dump["session"] and dump["session"]["audio"]: + if "input" in dump["session"]["audio"] and dump["session"]["audio"]["input"]: + if "turn_detection" in dump["session"]["audio"]["input"]: + if dump["session"]["audio"]["input"]["turn_detection"] is False: + dump["session"]["audio"]["input"]["turn_detection"] = None + + return dump + + +class InputAudioBufferAppendEvent(ClientEvent): + """Event to append audio data to the input buffer. + + Parameters: + type: Event type, always "input_audio_buffer.append". + audio: Base64-encoded audio data to append. + """ + + type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" + audio: str # base64-encoded audio + + +class InputAudioBufferCommitEvent(ClientEvent): + """Event to commit the current input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.commit". + """ + + type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" + + +class InputAudioBufferClearEvent(ClientEvent): + """Event to clear the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.clear". + """ + + type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" + + +class ConversationItemCreateEvent(ClientEvent): + """Event to create a new conversation item. + + Parameters: + type: Event type, always "conversation.item.create". + previous_item_id: ID of the item to insert after, if any. + item: The conversation item to create. + """ + + type: Literal["conversation.item.create"] = "conversation.item.create" + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemTruncateEvent(ClientEvent): + """Event to truncate a conversation item's audio content. + + Parameters: + type: Event type, always "conversation.item.truncate". + item_id: ID of the item to truncate. + content_index: Index of the content to truncate within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + + type: Literal["conversation.item.truncate"] = "conversation.item.truncate" + item_id: str + content_index: int + audio_end_ms: int + + +class ConversationItemDeleteEvent(ClientEvent): + """Event to delete a conversation item. + + Parameters: + type: Event type, always "conversation.item.delete". + item_id: ID of the item to delete. + """ + + type: Literal["conversation.item.delete"] = "conversation.item.delete" + item_id: str + + +class ConversationItemRetrieveEvent(ClientEvent): + """Event to retrieve a conversation item by ID. + + Parameters: + type: Event type, always "conversation.item.retrieve". + item_id: ID of the item to retrieve. + """ + + type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" + item_id: str + + +class ResponseCreateEvent(ClientEvent): + """Event to create a new assistant response. + + Parameters: + type: Event type, always "response.create". + response: Optional response configuration properties. + """ + + type: Literal["response.create"] = "response.create" + response: Optional[ResponseProperties] = None + + +class ResponseCancelEvent(ClientEvent): + """Event to cancel the current assistant response. + + Parameters: + type: Event type, always "response.cancel". + """ + + type: Literal["response.cancel"] = "response.cancel" + + +# +# server events +# + + +class ServerEvent(BaseModel): + """Base class for server events received from the realtime API. + + Parameters: + event_id: Unique identifier for the event. + type: Type of the server event. + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + event_id: str + type: str + + +class SessionCreatedEvent(ServerEvent): + """Event indicating a session has been created. + + Parameters: + type: Event type, always "session.created". + session: The created session properties. + """ + + type: Literal["session.created"] + session: SessionProperties + + +class SessionUpdatedEvent(ServerEvent): + """Event indicating a session has been updated. + + Parameters: + type: Event type, always "session.updated". + session: The updated session properties. + """ + + type: Literal["session.updated"] + session: SessionProperties + + +class ConversationCreated(ServerEvent): + """Event indicating a conversation has been created. + + Parameters: + type: Event type, always "conversation.created". + conversation: The created conversation. + """ + + type: Literal["conversation.created"] + conversation: RealtimeConversation + + +class ConversationItemAdded(ServerEvent): + """Event indicating a conversation item has been added. + + Parameters: + type: Event type, always "conversation.item.added". + previous_item_id: ID of the previous item, if any. + item: The added conversation item. + """ + + type: Literal["conversation.item.added"] + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemDone(ServerEvent): + """Event indicating a conversation item is done processing. + + Parameters: + type: Event type, always "conversation.item.done". + previous_item_id: ID of the previous item, if any. + item: The completed conversation item. + """ + + type: Literal["conversation.item.done"] + previous_item_id: Optional[str] = None + item: ConversationItem + + +class ConversationItemInputAudioTranscriptionDelta(ServerEvent): + """Event containing incremental input audio transcription. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.delta". + item_id: ID of the conversation item being transcribed. + content_index: Index of the content within the item. + delta: Incremental transcription text. + """ + + type: Literal["conversation.item.input_audio_transcription.delta"] + item_id: str + content_index: int + delta: str + + +class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): + """Event indicating input audio transcription is complete. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.completed". + item_id: ID of the conversation item that was transcribed. + content_index: Index of the content within the item. + transcript: Complete transcription text. + """ + + type: Literal["conversation.item.input_audio_transcription.completed"] + item_id: str + content_index: int + transcript: str + + +class ConversationItemInputAudioTranscriptionFailed(ServerEvent): + """Event indicating input audio transcription failed. + + Parameters: + type: Event type, always "conversation.item.input_audio_transcription.failed". + item_id: ID of the conversation item that failed transcription. + content_index: Index of the content within the item. + error: Error details for the transcription failure. + """ + + type: Literal["conversation.item.input_audio_transcription.failed"] + item_id: str + content_index: int + error: RealtimeError + + +class ConversationItemTruncated(ServerEvent): + """Event indicating a conversation item has been truncated. + + Parameters: + type: Event type, always "conversation.item.truncated". + item_id: ID of the truncated conversation item. + content_index: Index of the content within the item. + audio_end_ms: End time in milliseconds for the truncated audio. + """ + + type: Literal["conversation.item.truncated"] + item_id: str + content_index: int + audio_end_ms: int + + +class ConversationItemDeleted(ServerEvent): + """Event indicating a conversation item has been deleted. + + Parameters: + type: Event type, always "conversation.item.deleted". + item_id: ID of the deleted conversation item. + """ + + type: Literal["conversation.item.deleted"] + item_id: str + + +class ConversationItemRetrieved(ServerEvent): + """Event containing a retrieved conversation item. + + Parameters: + type: Event type, always "conversation.item.retrieved". + item: The retrieved conversation item. + """ + + type: Literal["conversation.item.retrieved"] + item: ConversationItem + + +class ResponseCreated(ServerEvent): + """Event indicating an assistant response has been created. + + Parameters: + type: Event type, always "response.created". + response: The created response object. + """ + + type: Literal["response.created"] + response: "Response" + + +class ResponseDone(ServerEvent): + """Event indicating an assistant response is complete. + + Parameters: + type: Event type, always "response.done". + response: The completed response object. + """ + + type: Literal["response.done"] + response: "Response" + + +class ResponseOutputItemAdded(ServerEvent): + """Event indicating an output item has been added to a response. + + Parameters: + type: Event type, always "response.output_item.added". + response_id: ID of the response. + output_index: Index of the output item. + item: The added conversation item. + """ + + type: Literal["response.output_item.added"] + response_id: str + output_index: int + item: ConversationItem + + +class ResponseOutputItemDone(ServerEvent): + """Event indicating an output item is complete. + + Parameters: + type: Event type, always "response.output_item.done". + response_id: ID of the response. + output_index: Index of the output item. + item: The completed conversation item. + """ + + type: Literal["response.output_item.done"] + response_id: str + output_index: int + item: ConversationItem + + +class ResponseContentPartAdded(ServerEvent): + """Event indicating a content part has been added to a response. + + Parameters: + type: Event type, always "response.content_part.added". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The added content part. + """ + + type: Literal["response.content_part.added"] + response_id: str + item_id: str + output_index: int + content_index: int + part: ItemContent + + +class ResponseContentPartDone(ServerEvent): + """Event indicating a content part is complete. + + Parameters: + type: Event type, always "response.content_part.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + part: The completed content part. + """ + + type: Literal["response.content_part.done"] + response_id: str + item_id: str + output_index: int + content_index: int + part: ItemContent + + +class ResponseTextDelta(ServerEvent): + """Event containing incremental text from a response. + + Parameters: + type: Event type, always "response.output_text.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental text content. + """ + + type: Literal["response.output_text.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseTextDone(ServerEvent): + """Event indicating text content is complete. + + Parameters: + type: Event type, always "response.output_text.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + text: Complete text content. + """ + + type: Literal["response.output_text.done"] + response_id: str + item_id: str + output_index: int + content_index: int + text: str + + +class ResponseAudioTranscriptDelta(ServerEvent): + """Event containing incremental audio transcript from a response. + + Parameters: + type: Event type, always "response.output_audio_transcript.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Incremental transcript text. + """ + + type: Literal["response.output_audio_transcript.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str + + +class ResponseAudioTranscriptDone(ServerEvent): + """Event indicating audio transcript is complete. + + Parameters: + type: Event type, always "response.output_audio_transcript.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + transcript: Complete transcript text. + """ + + type: Literal["response.output_audio_transcript.done"] + response_id: str + item_id: str + output_index: int + content_index: int + transcript: str + + +class ResponseAudioDelta(ServerEvent): + """Event containing incremental audio data from a response. + + Parameters: + type: Event type, always "response.output_audio.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + delta: Base64-encoded incremental audio data. + """ + + type: Literal["response.output_audio.delta"] + response_id: str + item_id: str + output_index: int + content_index: int + delta: str # base64-encoded audio + + +class ResponseAudioDone(ServerEvent): + """Event indicating audio content is complete. + + Parameters: + type: Event type, always "response.output_audio.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + content_index: Index of the content part. + """ + + type: Literal["response.output_audio.done"] + response_id: str + item_id: str + output_index: int + content_index: int + + +class ResponseFunctionCallArgumentsDelta(ServerEvent): + """Event containing incremental function call arguments. + + Parameters: + type: Event type, always "response.function_call_arguments.delta". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + delta: Incremental function arguments as JSON. + """ + + type: Literal["response.function_call_arguments.delta"] + response_id: str + item_id: str + output_index: int + call_id: str + delta: str + + +class ResponseFunctionCallArgumentsDone(ServerEvent): + """Event indicating function call arguments are complete. + + Parameters: + type: Event type, always "response.function_call_arguments.done". + response_id: ID of the response. + item_id: ID of the conversation item. + output_index: Index of the output item. + call_id: ID of the function call. + arguments: Complete function arguments as JSON string. + """ + + type: Literal["response.function_call_arguments.done"] + response_id: str + item_id: str + output_index: int + call_id: str + arguments: str + + +class InputAudioBufferSpeechStarted(ServerEvent): + """Event indicating speech has started in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_started". + audio_start_ms: Start time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + + type: Literal["input_audio_buffer.speech_started"] + audio_start_ms: int + item_id: str + + +class InputAudioBufferSpeechStopped(ServerEvent): + """Event indicating speech has stopped in the input audio buffer. + + Parameters: + type: Event type, always "input_audio_buffer.speech_stopped". + audio_end_ms: End time of speech in milliseconds. + item_id: ID of the associated conversation item. + """ + + type: Literal["input_audio_buffer.speech_stopped"] + audio_end_ms: int + item_id: str + + +class InputAudioBufferCommitted(ServerEvent): + """Event indicating the input audio buffer has been committed. + + Parameters: + type: Event type, always "input_audio_buffer.committed". + previous_item_id: ID of the previous item, if any. + item_id: ID of the committed conversation item. + """ + + type: Literal["input_audio_buffer.committed"] + previous_item_id: Optional[str] = None + item_id: str + + +class InputAudioBufferCleared(ServerEvent): + """Event indicating the input audio buffer has been cleared. + + Parameters: + type: Event type, always "input_audio_buffer.cleared". + """ + + type: Literal["input_audio_buffer.cleared"] + + +class ErrorEvent(ServerEvent): + """Event indicating an error occurred. + + Parameters: + type: Event type, always "error". + error: Error details. + """ + + type: Literal["error"] + error: RealtimeError + + +class RateLimitsUpdated(ServerEvent): + """Event indicating rate limits have been updated. + + Parameters: + type: Event type, always "rate_limits.updated". + rate_limits: List of rate limit information. + """ + + type: Literal["rate_limits.updated"] + rate_limits: List[Dict[str, Any]] + + +class CachedTokensDetails(BaseModel): + """Details about cached tokens. + + Parameters: + text_tokens: Number of cached text tokens. + audio_tokens: Number of cached audio tokens. + """ + + text_tokens: Optional[int] = 0 + audio_tokens: Optional[int] = 0 + + +class TokenDetails(BaseModel): + """Detailed token usage information. + + Parameters: + cached_tokens: Number of cached tokens used. Defaults to 0. + text_tokens: Number of text tokens used. Defaults to 0. + audio_tokens: Number of audio tokens used. Defaults to 0. + cached_tokens_details: Detailed breakdown of cached tokens. + image_tokens: Number of image tokens used (for input only). + """ + + cached_tokens: Optional[int] = 0 + text_tokens: Optional[int] = 0 + audio_tokens: Optional[int] = 0 + cached_tokens_details: Optional[CachedTokensDetails] = None + image_tokens: Optional[int] = 0 + + class Config: + """Pydantic configuration for TokenDetails.""" + + extra = "allow" + + +class Usage(BaseModel): + """Token usage statistics for a response. + + Parameters: + total_tokens: Total number of tokens used. + input_tokens: Number of input tokens used. + output_tokens: Number of output tokens used. + input_token_details: Detailed breakdown of input token usage. + output_token_details: Detailed breakdown of output token usage. + """ + + total_tokens: int + input_tokens: int + output_tokens: int + input_token_details: TokenDetails + output_token_details: TokenDetails + + +class Response(BaseModel): + """A complete assistant response. + + Parameters: + id: Unique identifier for the response. + object: Object type, always "realtime.response". + status: Current status of the response. + status_details: Additional status information. + output: List of conversation items in the response. + conversation_id: Which conversation the response is added to. + output_modalities: The set of modalities the model used to respond. + max_output_tokens: Maximum number of output tokens used. + audio: Audio configuration for the response. + usage: Token usage statistics for the response. + voice: The voice the model used to respond. + temperature: Sampling temperature used for the response. + output_audio_format: The format of output audio. + """ + + id: str + object: Literal["realtime.response"] + status: Literal["completed", "in_progress", "incomplete", "cancelled", "failed"] + status_details: Any + output: List[ConversationItem] + output_modalities: Optional[List[Literal["text", "audio"]]] = None + max_output_tokens: Optional[Union[int, Literal["inf"]]] = None + audio: Optional[AudioConfiguration] = None + usage: Optional[Usage] = None + voice: Optional[str] = None + temperature: Optional[float] = None + output_audio_format: Optional[str] = None + + +_server_event_types = { + "error": ErrorEvent, + "session.created": SessionCreatedEvent, + "session.updated": SessionUpdatedEvent, + "conversation.created": ConversationCreated, + "input_audio_buffer.committed": InputAudioBufferCommitted, + "input_audio_buffer.cleared": InputAudioBufferCleared, + "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, + "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, + "conversation.item.added": ConversationItemAdded, + "conversation.item.done": ConversationItemDone, + "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, + "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, + "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, + "conversation.item.truncated": ConversationItemTruncated, + "conversation.item.deleted": ConversationItemDeleted, + "conversation.item.retrieved": ConversationItemRetrieved, + "response.created": ResponseCreated, + "response.done": ResponseDone, + "response.output_item.added": ResponseOutputItemAdded, + "response.output_item.done": ResponseOutputItemDone, + "response.content_part.added": ResponseContentPartAdded, + "response.content_part.done": ResponseContentPartDone, + "response.output_text.delta": ResponseTextDelta, + "response.output_text.done": ResponseTextDone, + "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta, + "response.output_audio_transcript.done": ResponseAudioTranscriptDone, + "response.output_audio.delta": ResponseAudioDelta, + "response.output_audio.done": ResponseAudioDone, + "response.function_call_arguments.delta": ResponseFunctionCallArgumentsDelta, + "response.function_call_arguments.done": ResponseFunctionCallArgumentsDone, + "rate_limits.updated": RateLimitsUpdated, +} + + +def parse_server_event(str): + """Parse a server event from JSON string. + + Args: + str: JSON string containing the server event. + + Returns: + Parsed server event object of the appropriate type. + + Raises: + Exception: If the event type is unimplemented or parsing fails. + """ + try: + event = json.loads(str) + event_type = event["type"] + if event_type not in _server_event_types: + raise Exception(f"Unimplemented server event type: {event_type}") + return _server_event_types[event_type].model_validate(event) + except Exception as e: + raise Exception(f"{e} \n\n{str}") diff --git a/src/pipecat/services/openai/realtime/frames.py b/src/pipecat/services/openai/realtime/frames.py new file mode 100644 index 000000000..8617c6efd --- /dev/null +++ b/src/pipecat/services/openai/realtime/frames.py @@ -0,0 +1,37 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +"""Custom frame types for OpenAI Realtime API integration.""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from pipecat.frames.frames import DataFrame, FunctionCallResultFrame + +if TYPE_CHECKING: + from pipecat.services.openai.realtime.context import OpenAIRealtimeLLMContext + + +@dataclass +class RealtimeMessagesUpdateFrame(DataFrame): + """Frame indicating that the realtime context messages have been updated. + + Parameters: + context: The updated OpenAI realtime LLM context. + """ + + context: "OpenAIRealtimeLLMContext" + + +@dataclass +class RealtimeFunctionCallResultFrame(DataFrame): + """Frame containing function call results for the realtime service. + + Parameters: + result_frame: The function call result frame to send to the realtime API. + """ + + result_frame: FunctionCallResultFrame diff --git a/src/pipecat/services/openai_realtime/openai.py b/src/pipecat/services/openai/realtime/llm.py similarity index 100% rename from src/pipecat/services/openai_realtime/openai.py rename to src/pipecat/services/openai/realtime/llm.py diff --git a/src/pipecat/services/openai_realtime/__init__.py b/src/pipecat/services/openai_realtime/__init__.py index 6f3154f1c..f1fbefeda 100644 --- a/src/pipecat/services/openai_realtime/__init__.py +++ b/src/pipecat/services/openai_realtime/__init__.py @@ -1,9 +1,27 @@ -from .azure import AzureRealtimeLLMService -from .events import ( +# +# Copyright (c) 2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + +import warnings + +from pipecat.services.azure.realtime.llm import AzureRealtimeLLMService +from pipecat.services.openai.realtime.events import ( InputAudioNoiseReduction, InputAudioTranscription, SemanticTurnDetection, SessionProperties, TurnDetection, ) -from .openai import OpenAIRealtimeLLMService +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/azure.py b/src/pipecat/services/openai_realtime/azure.py index aed53b94c..dae6ef496 100644 --- a/src/pipecat/services/openai_realtime/azure.py +++ b/src/pipecat/services/openai_realtime/azure.py @@ -1,67 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Azure OpenAI Realtime LLM service implementation.""" -from loguru import logger +import warnings -from .openai import OpenAIRealtimeLLMService +from pipecat.services.azure.realtime.llm import * -try: - from websockets.asyncio.client import connect as websocket_connect -except ModuleNotFoundError as e: - logger.error(f"Exception: {e}") - logger.error( - "In order to use OpenAI, you need to `pip install pipecat-ai[openai]`. Also, set `OPENAI_API_KEY` environment variable." +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.azure are deprecated. " + "Please use the equivalent types from " + "pipecat.services.azure.realtime.llm instead.", + DeprecationWarning, + stacklevel=2, ) - raise Exception(f"Missing module: {e}") - - -class AzureRealtimeLLMService(OpenAIRealtimeLLMService): - """Azure OpenAI Realtime LLM service with Azure-specific authentication. - - Extends the OpenAI Realtime service to work with Azure OpenAI endpoints, - using Azure's authentication headers and endpoint format. Provides the same - real-time audio and text communication capabilities as the base OpenAI service. - """ - - def __init__( - self, - *, - api_key: str, - base_url: str, - **kwargs, - ): - """Initialize Azure Realtime LLM service. - - Args: - api_key: The API key for the Azure OpenAI service. - base_url: The full Azure WebSocket endpoint URL including api-version and deployment. - Example: "wss://my-project.openai.azure.com/openai/realtime?api-version=2024-10-01-preview&deployment=my-realtime-deployment" - **kwargs: Additional arguments passed to parent OpenAIRealtimeLLMService. - """ - super().__init__(base_url=base_url, api_key=api_key, **kwargs) - self.api_key = api_key - self.base_url = base_url - - async def _connect(self): - try: - if self._websocket: - # Here we assume that if we have a websocket, we are connected. We - # handle disconnections in the send/recv code paths. - return - - logger.info(f"Connecting to {self.base_url}, api key: {self.api_key}") - self._websocket = await websocket_connect( - uri=self.base_url, - additional_headers={ - "api-key": self.api_key, - }, - ) - self._receive_task = self.create_task(self._receive_task_handler()) - except Exception as e: - logger.error(f"{self} initialization error: {e}") - self._websocket = None diff --git a/src/pipecat/services/openai_realtime/context.py b/src/pipecat/services/openai_realtime/context.py index cb1c0a9f5..58f1cfe75 100644 --- a/src/pipecat/services/openai_realtime/context.py +++ b/src/pipecat/services/openai_realtime/context.py @@ -1,272 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """OpenAI Realtime LLM context and aggregator implementations.""" -import copy -import json +import warnings -from loguru import logger +from pipecat.services.openai.realtime.context import * -from pipecat.frames.frames import ( - Frame, - FunctionCallResultFrame, - InterimTranscriptionFrame, - LLMMessagesUpdateFrame, - LLMSetToolsFrame, - LLMTextFrame, - TranscriptionFrame, -) -from pipecat.processors.aggregators.openai_llm_context import OpenAILLMContext -from pipecat.processors.frame_processor import FrameDirection -from pipecat.services.openai.llm import ( - OpenAIAssistantContextAggregator, - OpenAIUserContextAggregator, -) - -from . import events -from .frames import RealtimeFunctionCallResultFrame, RealtimeMessagesUpdateFrame - - -class OpenAIRealtimeLLMContext(OpenAILLMContext): - """OpenAI Realtime LLM context with session management and message conversion. - - Extends the standard OpenAI LLM context to support real-time session properties, - instruction management, and conversion between standard message formats and - realtime conversation items. - """ - - def __init__(self, messages=None, tools=None, **kwargs): - """Initialize the OpenAIRealtimeLLMContext. - - Args: - messages: Initial conversation messages. Defaults to None. - tools: Available function tools. Defaults to None. - **kwargs: Additional arguments passed to parent OpenAILLMContext. - """ - super().__init__(messages=messages, tools=tools, **kwargs) - self.__setup_local() - - def __setup_local(self): - self.llm_needs_settings_update = True - self.llm_needs_initial_messages = True - self._session_instructions = "" - - return - - @staticmethod - def upgrade_to_realtime(obj: OpenAILLMContext) -> "OpenAIRealtimeLLMContext": - """Upgrade a standard OpenAI LLM context to a realtime context. - - Args: - obj: The OpenAILLMContext instance to upgrade. - - Returns: - The upgraded OpenAIRealtimeLLMContext instance. - """ - if isinstance(obj, OpenAILLMContext) and not isinstance(obj, OpenAIRealtimeLLMContext): - obj.__class__ = OpenAIRealtimeLLMContext - obj.__setup_local() - return obj - - # todo - # - finish implementing all frames - - def from_standard_message(self, message): - """Convert a standard message format to a realtime conversation item. - - Args: - message: The standard message dictionary to convert. - - Returns: - A ConversationItem instance for the realtime API. - """ - if message.get("role") == "user": - content = message.get("content") - if isinstance(message.get("content"), list): - content = "" - for c in message.get("content"): - if c.get("type") == "text": - content += " " + c.get("text") - else: - logger.error( - f"Unhandled content type in context message: {c.get('type')} - {message}" - ) - return events.ConversationItem( - role="user", - type="message", - content=[events.ItemContent(type="input_text", text=content)], - ) - if message.get("role") == "assistant" and message.get("tool_calls"): - tc = message.get("tool_calls")[0] - return events.ConversationItem( - type="function_call", - call_id=tc["id"], - name=tc["function"]["name"], - arguments=tc["function"]["arguments"], - ) - logger.error(f"Unhandled message type in from_standard_message: {message}") - - def get_messages_for_initializing_history(self): - """Get conversation items for initializing the realtime session history. - - Converts the context's messages to a format suitable for the realtime API, - handling system instructions and conversation history packaging. - - Returns: - List of conversation items for session initialization. - """ - # We can't load a long conversation history into the openai realtime api yet. (The API/model - # forgets that it can do audio, if you do a series of `conversation.item.create` calls.) So - # our general strategy until this is fixed is just to put everything into a first "user" - # message as a single input. - if not self.messages: - return [] - - messages = copy.deepcopy(self.messages) - - # If we have a "system" message as our first message, let's pull that out into session - # "instructions" - if messages[0].get("role") == "system": - self.llm_needs_settings_update = True - system = messages.pop(0) - content = system.get("content") - if isinstance(content, str): - self._session_instructions = content - elif isinstance(content, list): - self._session_instructions = content[0].get("text") - if not messages: - return [] - - # If we have just a single "user" item, we can just send it normally - if len(messages) == 1 and messages[0].get("role") == "user": - return [self.from_standard_message(messages[0])] - - # Otherwise, let's pack everything into a single "user" message with a bit of - # explanation for the LLM - intro_text = """ - This is a previously saved conversation. Please treat this conversation history as a - starting point for the current conversation.""" - - trailing_text = """ - This is the end of the previously saved conversation. Please continue the conversation - from here. If the last message is a user instruction or question, act on that instruction - or answer the question. If the last message is an assistant response, simple say that you - are ready to continue the conversation.""" - - return [ - { - "role": "user", - "type": "message", - "content": [ - { - "type": "input_text", - "text": "\n\n".join( - [intro_text, json.dumps(messages, indent=2), trailing_text] - ), - } - ], - } - ] - - def add_user_content_item_as_message(self, item): - """Add a user content item as a standard message to the context. - - Args: - item: The conversation item to add as a user message. - """ - message = { - "role": "user", - "content": [{"type": "text", "text": item.content[0].transcript}], - } - self.add_message(message) - - -class OpenAIRealtimeUserContextAggregator(OpenAIUserContextAggregator): - """User context aggregator for OpenAI Realtime API. - - Handles user input frames and generates appropriate context updates - for the realtime conversation, including message updates and tool settings. - - Args: - context: The OpenAI realtime LLM context. - **kwargs: Additional arguments passed to parent aggregator. - """ - - async def process_frame( - self, frame: Frame, direction: FrameDirection = FrameDirection.DOWNSTREAM - ): - """Process incoming frames and handle realtime-specific frame types. - - Args: - frame: The frame to process. - direction: The direction of frame flow in the pipeline. - """ - await super().process_frame(frame, direction) - # Parent does not push LLMMessagesUpdateFrame. This ensures that in a typical pipeline, - # messages are only processed by the user context aggregator, which is generally what we want. But - # we also need to send new messages over the websocket, so the openai realtime API has them - # in its context. - if isinstance(frame, LLMMessagesUpdateFrame): - await self.push_frame(RealtimeMessagesUpdateFrame(context=self._context)) - - # Parent also doesn't push the LLMSetToolsFrame. - if isinstance(frame, LLMSetToolsFrame): - await self.push_frame(frame, direction) - - async def push_aggregation(self): - """Push user input aggregation. - - Currently ignores all user input coming into the pipeline as realtime - audio input is handled directly by the service. - """ - # for the moment, ignore all user input coming into the pipeline. - # todo: think about whether/how to fix this to allow for text input from - # upstream (transport/transcription, or other sources) - pass - - -class OpenAIRealtimeAssistantContextAggregator(OpenAIAssistantContextAggregator): - """Assistant context aggregator for OpenAI Realtime API. - - Handles assistant output frames from the realtime service, filtering - out duplicate text frames and managing function call results. - - Args: - context: The OpenAI realtime LLM context. - **kwargs: Additional arguments passed to parent aggregator. - """ - - # The LLMAssistantContextAggregator uses TextFrames to aggregate the LLM output, - # but the OpenAIRealtimeLLMService pushes LLMTextFrames and TTSTextFrames. We - # need to override this proces_frame for LLMTextFrame, so that only the TTSTextFrames - # are process. This ensures that the context gets only one set of messages. - # OpenAIRealtimeLLMService also pushes TranscriptionFrames and InterimTranscriptionFrames, - # so we need to ignore pushing those as well, as they're also TextFrames. - async def process_frame(self, frame: Frame, direction: FrameDirection): - """Process assistant frames, filtering out duplicate text content. - - Args: - frame: The frame to process. - direction: The direction of frame flow in the pipeline. - """ - if not isinstance(frame, (LLMTextFrame, TranscriptionFrame, InterimTranscriptionFrame)): - await super().process_frame(frame, direction) - - async def handle_function_call_result(self, frame: FunctionCallResultFrame): - """Handle function call result and notify the realtime service. - - Args: - frame: The function call result frame to handle. - """ - await super().handle_function_call_result(frame) - - # The standard function callback code path pushes the FunctionCallResultFrame from the llm itself, - # so we didn't have a chance to add the result to the openai realtime api context. Let's push a - # special frame to do that. - await self.push_frame( - RealtimeFunctionCallResultFrame(result_frame=frame), FrameDirection.UPSTREAM - ) +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.context are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.context instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/events.py b/src/pipecat/services/openai_realtime/events.py index 200e59d68..53b4b0dff 100644 --- a/src/pipecat/services/openai_realtime/events.py +++ b/src/pipecat/services/openai_realtime/events.py @@ -1,1106 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Event models and data structures for OpenAI Realtime API communication.""" -import json -import uuid -from typing import Any, Dict, List, Literal, Optional, Union - -from pydantic import BaseModel, ConfigDict, Field - -# -# session properties -# - - -class AudioFormat(BaseModel): - """Base class for audio format configuration.""" - - type: str - - -class PCMAudioFormat(AudioFormat): - """PCM audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcm". - rate: Sample rate, always 24000 for PCM. - """ - - type: Literal["audio/pcm"] = "audio/pcm" - rate: Literal[24000] = 24000 - - -class PCMUAudioFormat(AudioFormat): - """PCMU (G.711 μ-law) audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcmu". - """ - - type: Literal["audio/pcmu"] = "audio/pcmu" - - -class PCMAAudioFormat(AudioFormat): - """PCMA (G.711 A-law) audio format configuration. - - Parameters: - type: Audio format type, always "audio/pcma". - """ - - type: Literal["audio/pcma"] = "audio/pcma" - - -class InputAudioTranscription(BaseModel): - """Configuration for audio transcription settings.""" - - model: str = "gpt-4o-transcribe" - language: Optional[str] - prompt: Optional[str] - - def __init__( - self, - model: Optional[str] = "gpt-4o-transcribe", - language: Optional[str] = None, - prompt: Optional[str] = None, - ): - """Initialize InputAudioTranscription. - - Args: - model: Transcription model to use (e.g., "gpt-4o-transcribe", "whisper-1"). - language: Optional language code for transcription. - prompt: Optional transcription hint text. - """ - super().__init__(model=model, language=language, prompt=prompt) - - -class TurnDetection(BaseModel): - """Server-side voice activity detection configuration. - - Parameters: - type: Detection type, must be "server_vad". - threshold: Voice activity detection threshold (0.0-1.0). Defaults to 0.5. - prefix_padding_ms: Padding before speech starts in milliseconds. Defaults to 300. - silence_duration_ms: Silence duration to detect speech end in milliseconds. Defaults to 500. - """ - - type: Optional[Literal["server_vad"]] = "server_vad" - threshold: Optional[float] = 0.5 - prefix_padding_ms: Optional[int] = 300 - silence_duration_ms: Optional[int] = 500 - - -class SemanticTurnDetection(BaseModel): - """Semantic-based turn detection configuration. - - Parameters: - type: Detection type, must be "semantic_vad". - eagerness: Turn detection eagerness level. Can be "low", "medium", "high", or "auto". - create_response: Whether to automatically create responses on turn detection. - interrupt_response: Whether to interrupt ongoing responses on turn detection. - """ - - type: Optional[Literal["semantic_vad"]] = "semantic_vad" - eagerness: Optional[Literal["low", "medium", "high", "auto"]] = None - create_response: Optional[bool] = None - interrupt_response: Optional[bool] = None - - -class InputAudioNoiseReduction(BaseModel): - """Input audio noise reduction configuration. - - Parameters: - type: Noise reduction type for different microphone scenarios. - """ - - type: Optional[Literal["near_field", "far_field"]] - - -class AudioInput(BaseModel): - """Audio input configuration. - - Parameters: - format: The format of the input audio. - transcription: Configuration for input audio transcription. - noise_reduction: Configuration for input audio noise reduction. - turn_detection: Configuration for turn detection, or False to disable. - """ - - format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None - transcription: Optional[InputAudioTranscription] = None - noise_reduction: Optional[InputAudioNoiseReduction] = None - turn_detection: Optional[Union[TurnDetection, SemanticTurnDetection, bool]] = None - - -class AudioOutput(BaseModel): - """Audio output configuration. - - Parameters: - format: The format of the output audio. - voice: The voice the model uses to respond. - speed: The speed of the model's spoken response. - """ - - format: Optional[Union[PCMAudioFormat, PCMUAudioFormat, PCMAAudioFormat]] = None - voice: Optional[str] = None - speed: Optional[float] = None - - -class AudioConfiguration(BaseModel): - """Audio configuration for input and output. - - Parameters: - input: Configuration for input audio. - output: Configuration for output audio. - """ - - input: Optional[AudioInput] = None - output: Optional[AudioOutput] = None - - -class SessionProperties(BaseModel): - """Configuration properties for an OpenAI Realtime session. - - Parameters: - type: The type of session, always "realtime". - object: Object type identifier, always "realtime.session". - id: Unique identifier for the session. - model: The Realtime model used for this session. - output_modalities: The set of modalities the model can respond with. - instructions: System instructions for the assistant. - audio: Configuration for input and output audio. - tools: Available function tools for the assistant. - tool_choice: Tool usage strategy ("auto", "none", or "required"). - max_output_tokens: Maximum tokens in response or "inf" for unlimited. - tracing: Configuration options for tracing. - prompt: Reference to a prompt template and its variables. - expires_at: Session expiration timestamp. - include: Additional fields to include in server outputs. - """ - - type: Optional[Literal["realtime"]] = "realtime" - object: Optional[Literal["realtime.session"]] = None - id: Optional[str] = None - model: Optional[str] = None - output_modalities: Optional[List[Literal["text", "audio"]]] = None - instructions: Optional[str] = None - audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None - tool_choice: Optional[Literal["auto", "none", "required"]] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - tracing: Optional[Union[Literal["auto"], Dict]] = None - prompt: Optional[Dict] = None - expires_at: Optional[int] = None - include: Optional[List[str]] = None - - -# -# context -# - - -class ItemContent(BaseModel): - """Content within a conversation item. - - Parameters: - type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). - text: Text content for text-based items. - audio: Base64-encoded audio data for audio items. - transcript: Transcribed text for audio items. - """ - - type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] - text: Optional[str] = None - audio: Optional[str] = None # base64-encoded audio - transcript: Optional[str] = None - - -class ConversationItem(BaseModel): - """A conversation item in the realtime session. - - Parameters: - id: Unique identifier for the item, auto-generated if not provided. - object: Object type identifier for the realtime API. - type: Item type (message, function_call, or function_call_output). - status: Current status of the item. - role: Speaker role for message items (user, assistant, or system). - content: Content list for message items. - call_id: Function call identifier for function_call items. - name: Function name for function_call items. - arguments: Function arguments as JSON string for function_call items. - output: Function output as JSON string for function_call_output items. - """ - - id: str = Field(default_factory=lambda: str(uuid.uuid4().hex)) - object: Optional[Literal["realtime.item"]] = None - type: Literal["message", "function_call", "function_call_output"] - status: Optional[Literal["completed", "in_progress", "incomplete"]] = None - # role and content are present for message items - role: Optional[Literal["user", "assistant", "system"]] = None - content: Optional[List[ItemContent]] = None - # these four fields are present for function_call items - call_id: Optional[str] = None - name: Optional[str] = None - arguments: Optional[str] = None - output: Optional[str] = None - - -class RealtimeConversation(BaseModel): - """A realtime conversation session. - - Parameters: - id: Unique identifier for the conversation. - object: Object type identifier, always "realtime.conversation". - """ - - id: str - object: Literal["realtime.conversation"] - - -class ResponseProperties(BaseModel): - """Properties for configuring assistant responses. - - Parameters: - output_modalities: Output modalities for the response. Must be either ["text"] or ["audio"]. Defaults to ["audio"]. - instructions: Specific instructions for this response. - audio: Audio configuration for this response. - tools: Available tools for this response. - tool_choice: Tool usage strategy for this response. - temperature: Sampling temperature for this response. - max_output_tokens: Maximum tokens for this response. - """ - - output_modalities: Optional[List[Literal["text", "audio"]]] = ["audio"] - instructions: Optional[str] = None - audio: Optional[AudioConfiguration] = None - tools: Optional[List[Dict]] = None - tool_choice: Optional[Literal["auto", "none", "required"]] = None - temperature: Optional[float] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - - -# -# error class -# -class RealtimeError(BaseModel): - """Error information from the realtime API. - - Parameters: - type: Error type identifier. - code: Specific error code. - message: Human-readable error message. - param: Parameter name that caused the error, if applicable. - event_id: Event ID associated with the error, if applicable. - """ - - type: str - code: Optional[str] = "" - message: str - param: Optional[str] = None - event_id: Optional[str] = None - - -# -# client events -# - - -class ClientEvent(BaseModel): - """Base class for client events sent to the realtime API. - - Parameters: - event_id: Unique identifier for the event, auto-generated if not provided. - """ - - event_id: str = Field(default_factory=lambda: str(uuid.uuid4())) - - -class SessionUpdateEvent(ClientEvent): - """Event to update session properties. - - Parameters: - type: Event type, always "session.update". - session: Updated session properties. - """ - - type: Literal["session.update"] = "session.update" - session: SessionProperties - - def model_dump(self, *args, **kwargs) -> Dict[str, Any]: - """Serialize the event to a dictionary. - - Handles special serialization for turn_detection where False becomes null. - - Args: - *args: Positional arguments passed to parent model_dump. - **kwargs: Keyword arguments passed to parent model_dump. - - Returns: - Dictionary representation of the event. - """ - dump = super().model_dump(*args, **kwargs) - - # Handle turn_detection in audio.input so that False becomes null - if "audio" in dump["session"] and dump["session"]["audio"]: - if "input" in dump["session"]["audio"] and dump["session"]["audio"]["input"]: - if "turn_detection" in dump["session"]["audio"]["input"]: - if dump["session"]["audio"]["input"]["turn_detection"] is False: - dump["session"]["audio"]["input"]["turn_detection"] = None - - return dump - - -class InputAudioBufferAppendEvent(ClientEvent): - """Event to append audio data to the input buffer. - - Parameters: - type: Event type, always "input_audio_buffer.append". - audio: Base64-encoded audio data to append. - """ - - type: Literal["input_audio_buffer.append"] = "input_audio_buffer.append" - audio: str # base64-encoded audio - - -class InputAudioBufferCommitEvent(ClientEvent): - """Event to commit the current input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.commit". - """ - - type: Literal["input_audio_buffer.commit"] = "input_audio_buffer.commit" - - -class InputAudioBufferClearEvent(ClientEvent): - """Event to clear the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.clear". - """ - - type: Literal["input_audio_buffer.clear"] = "input_audio_buffer.clear" - - -class ConversationItemCreateEvent(ClientEvent): - """Event to create a new conversation item. - - Parameters: - type: Event type, always "conversation.item.create". - previous_item_id: ID of the item to insert after, if any. - item: The conversation item to create. - """ - - type: Literal["conversation.item.create"] = "conversation.item.create" - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemTruncateEvent(ClientEvent): - """Event to truncate a conversation item's audio content. - - Parameters: - type: Event type, always "conversation.item.truncate". - item_id: ID of the item to truncate. - content_index: Index of the content to truncate within the item. - audio_end_ms: End time in milliseconds for the truncated audio. - """ - - type: Literal["conversation.item.truncate"] = "conversation.item.truncate" - item_id: str - content_index: int - audio_end_ms: int - - -class ConversationItemDeleteEvent(ClientEvent): - """Event to delete a conversation item. - - Parameters: - type: Event type, always "conversation.item.delete". - item_id: ID of the item to delete. - """ - - type: Literal["conversation.item.delete"] = "conversation.item.delete" - item_id: str - - -class ConversationItemRetrieveEvent(ClientEvent): - """Event to retrieve a conversation item by ID. - - Parameters: - type: Event type, always "conversation.item.retrieve". - item_id: ID of the item to retrieve. - """ - - type: Literal["conversation.item.retrieve"] = "conversation.item.retrieve" - item_id: str - - -class ResponseCreateEvent(ClientEvent): - """Event to create a new assistant response. - - Parameters: - type: Event type, always "response.create". - response: Optional response configuration properties. - """ - - type: Literal["response.create"] = "response.create" - response: Optional[ResponseProperties] = None - - -class ResponseCancelEvent(ClientEvent): - """Event to cancel the current assistant response. - - Parameters: - type: Event type, always "response.cancel". - """ - - type: Literal["response.cancel"] = "response.cancel" - - -# -# server events -# - - -class ServerEvent(BaseModel): - """Base class for server events received from the realtime API. - - Parameters: - event_id: Unique identifier for the event. - type: Type of the server event. - """ - - model_config = ConfigDict(arbitrary_types_allowed=True) - - event_id: str - type: str - - -class SessionCreatedEvent(ServerEvent): - """Event indicating a session has been created. - - Parameters: - type: Event type, always "session.created". - session: The created session properties. - """ - - type: Literal["session.created"] - session: SessionProperties - - -class SessionUpdatedEvent(ServerEvent): - """Event indicating a session has been updated. - - Parameters: - type: Event type, always "session.updated". - session: The updated session properties. - """ - - type: Literal["session.updated"] - session: SessionProperties - - -class ConversationCreated(ServerEvent): - """Event indicating a conversation has been created. - - Parameters: - type: Event type, always "conversation.created". - conversation: The created conversation. - """ - - type: Literal["conversation.created"] - conversation: RealtimeConversation - - -class ConversationItemAdded(ServerEvent): - """Event indicating a conversation item has been added. - - Parameters: - type: Event type, always "conversation.item.added". - previous_item_id: ID of the previous item, if any. - item: The added conversation item. - """ - - type: Literal["conversation.item.added"] - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemDone(ServerEvent): - """Event indicating a conversation item is done processing. - - Parameters: - type: Event type, always "conversation.item.done". - previous_item_id: ID of the previous item, if any. - item: The completed conversation item. - """ - - type: Literal["conversation.item.done"] - previous_item_id: Optional[str] = None - item: ConversationItem - - -class ConversationItemInputAudioTranscriptionDelta(ServerEvent): - """Event containing incremental input audio transcription. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.delta". - item_id: ID of the conversation item being transcribed. - content_index: Index of the content within the item. - delta: Incremental transcription text. - """ - - type: Literal["conversation.item.input_audio_transcription.delta"] - item_id: str - content_index: int - delta: str - - -class ConversationItemInputAudioTranscriptionCompleted(ServerEvent): - """Event indicating input audio transcription is complete. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.completed". - item_id: ID of the conversation item that was transcribed. - content_index: Index of the content within the item. - transcript: Complete transcription text. - """ - - type: Literal["conversation.item.input_audio_transcription.completed"] - item_id: str - content_index: int - transcript: str - - -class ConversationItemInputAudioTranscriptionFailed(ServerEvent): - """Event indicating input audio transcription failed. - - Parameters: - type: Event type, always "conversation.item.input_audio_transcription.failed". - item_id: ID of the conversation item that failed transcription. - content_index: Index of the content within the item. - error: Error details for the transcription failure. - """ - - type: Literal["conversation.item.input_audio_transcription.failed"] - item_id: str - content_index: int - error: RealtimeError - - -class ConversationItemTruncated(ServerEvent): - """Event indicating a conversation item has been truncated. - - Parameters: - type: Event type, always "conversation.item.truncated". - item_id: ID of the truncated conversation item. - content_index: Index of the content within the item. - audio_end_ms: End time in milliseconds for the truncated audio. - """ - - type: Literal["conversation.item.truncated"] - item_id: str - content_index: int - audio_end_ms: int - - -class ConversationItemDeleted(ServerEvent): - """Event indicating a conversation item has been deleted. - - Parameters: - type: Event type, always "conversation.item.deleted". - item_id: ID of the deleted conversation item. - """ - - type: Literal["conversation.item.deleted"] - item_id: str - - -class ConversationItemRetrieved(ServerEvent): - """Event containing a retrieved conversation item. - - Parameters: - type: Event type, always "conversation.item.retrieved". - item: The retrieved conversation item. - """ - - type: Literal["conversation.item.retrieved"] - item: ConversationItem - - -class ResponseCreated(ServerEvent): - """Event indicating an assistant response has been created. - - Parameters: - type: Event type, always "response.created". - response: The created response object. - """ - - type: Literal["response.created"] - response: "Response" - - -class ResponseDone(ServerEvent): - """Event indicating an assistant response is complete. - - Parameters: - type: Event type, always "response.done". - response: The completed response object. - """ - - type: Literal["response.done"] - response: "Response" - - -class ResponseOutputItemAdded(ServerEvent): - """Event indicating an output item has been added to a response. - - Parameters: - type: Event type, always "response.output_item.added". - response_id: ID of the response. - output_index: Index of the output item. - item: The added conversation item. - """ - - type: Literal["response.output_item.added"] - response_id: str - output_index: int - item: ConversationItem - - -class ResponseOutputItemDone(ServerEvent): - """Event indicating an output item is complete. - - Parameters: - type: Event type, always "response.output_item.done". - response_id: ID of the response. - output_index: Index of the output item. - item: The completed conversation item. - """ - - type: Literal["response.output_item.done"] - response_id: str - output_index: int - item: ConversationItem - - -class ResponseContentPartAdded(ServerEvent): - """Event indicating a content part has been added to a response. - - Parameters: - type: Event type, always "response.content_part.added". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - part: The added content part. - """ - - type: Literal["response.content_part.added"] - response_id: str - item_id: str - output_index: int - content_index: int - part: ItemContent - - -class ResponseContentPartDone(ServerEvent): - """Event indicating a content part is complete. - - Parameters: - type: Event type, always "response.content_part.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - part: The completed content part. - """ - - type: Literal["response.content_part.done"] - response_id: str - item_id: str - output_index: int - content_index: int - part: ItemContent - - -class ResponseTextDelta(ServerEvent): - """Event containing incremental text from a response. - - Parameters: - type: Event type, always "response.output_text.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Incremental text content. - """ - - type: Literal["response.output_text.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str - - -class ResponseTextDone(ServerEvent): - """Event indicating text content is complete. - - Parameters: - type: Event type, always "response.output_text.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - text: Complete text content. - """ - - type: Literal["response.output_text.done"] - response_id: str - item_id: str - output_index: int - content_index: int - text: str - - -class ResponseAudioTranscriptDelta(ServerEvent): - """Event containing incremental audio transcript from a response. - - Parameters: - type: Event type, always "response.output_audio_transcript.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Incremental transcript text. - """ - - type: Literal["response.output_audio_transcript.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str - - -class ResponseAudioTranscriptDone(ServerEvent): - """Event indicating audio transcript is complete. - - Parameters: - type: Event type, always "response.output_audio_transcript.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - transcript: Complete transcript text. - """ - - type: Literal["response.output_audio_transcript.done"] - response_id: str - item_id: str - output_index: int - content_index: int - transcript: str - - -class ResponseAudioDelta(ServerEvent): - """Event containing incremental audio data from a response. - - Parameters: - type: Event type, always "response.output_audio.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - delta: Base64-encoded incremental audio data. - """ - - type: Literal["response.output_audio.delta"] - response_id: str - item_id: str - output_index: int - content_index: int - delta: str # base64-encoded audio - - -class ResponseAudioDone(ServerEvent): - """Event indicating audio content is complete. - - Parameters: - type: Event type, always "response.output_audio.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - content_index: Index of the content part. - """ - - type: Literal["response.output_audio.done"] - response_id: str - item_id: str - output_index: int - content_index: int - - -class ResponseFunctionCallArgumentsDelta(ServerEvent): - """Event containing incremental function call arguments. - - Parameters: - type: Event type, always "response.function_call_arguments.delta". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - call_id: ID of the function call. - delta: Incremental function arguments as JSON. - """ - - type: Literal["response.function_call_arguments.delta"] - response_id: str - item_id: str - output_index: int - call_id: str - delta: str - - -class ResponseFunctionCallArgumentsDone(ServerEvent): - """Event indicating function call arguments are complete. - - Parameters: - type: Event type, always "response.function_call_arguments.done". - response_id: ID of the response. - item_id: ID of the conversation item. - output_index: Index of the output item. - call_id: ID of the function call. - arguments: Complete function arguments as JSON string. - """ - - type: Literal["response.function_call_arguments.done"] - response_id: str - item_id: str - output_index: int - call_id: str - arguments: str - - -class InputAudioBufferSpeechStarted(ServerEvent): - """Event indicating speech has started in the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.speech_started". - audio_start_ms: Start time of speech in milliseconds. - item_id: ID of the associated conversation item. - """ - - type: Literal["input_audio_buffer.speech_started"] - audio_start_ms: int - item_id: str - - -class InputAudioBufferSpeechStopped(ServerEvent): - """Event indicating speech has stopped in the input audio buffer. - - Parameters: - type: Event type, always "input_audio_buffer.speech_stopped". - audio_end_ms: End time of speech in milliseconds. - item_id: ID of the associated conversation item. - """ - - type: Literal["input_audio_buffer.speech_stopped"] - audio_end_ms: int - item_id: str - - -class InputAudioBufferCommitted(ServerEvent): - """Event indicating the input audio buffer has been committed. - - Parameters: - type: Event type, always "input_audio_buffer.committed". - previous_item_id: ID of the previous item, if any. - item_id: ID of the committed conversation item. - """ - - type: Literal["input_audio_buffer.committed"] - previous_item_id: Optional[str] = None - item_id: str - - -class InputAudioBufferCleared(ServerEvent): - """Event indicating the input audio buffer has been cleared. - - Parameters: - type: Event type, always "input_audio_buffer.cleared". - """ - - type: Literal["input_audio_buffer.cleared"] - - -class ErrorEvent(ServerEvent): - """Event indicating an error occurred. - - Parameters: - type: Event type, always "error". - error: Error details. - """ - - type: Literal["error"] - error: RealtimeError - - -class RateLimitsUpdated(ServerEvent): - """Event indicating rate limits have been updated. - - Parameters: - type: Event type, always "rate_limits.updated". - rate_limits: List of rate limit information. - """ - - type: Literal["rate_limits.updated"] - rate_limits: List[Dict[str, Any]] - - -class CachedTokensDetails(BaseModel): - """Details about cached tokens. - - Parameters: - text_tokens: Number of cached text tokens. - audio_tokens: Number of cached audio tokens. - """ - - text_tokens: Optional[int] = 0 - audio_tokens: Optional[int] = 0 - - -class TokenDetails(BaseModel): - """Detailed token usage information. - - Parameters: - cached_tokens: Number of cached tokens used. Defaults to 0. - text_tokens: Number of text tokens used. Defaults to 0. - audio_tokens: Number of audio tokens used. Defaults to 0. - cached_tokens_details: Detailed breakdown of cached tokens. - image_tokens: Number of image tokens used (for input only). - """ - - cached_tokens: Optional[int] = 0 - text_tokens: Optional[int] = 0 - audio_tokens: Optional[int] = 0 - cached_tokens_details: Optional[CachedTokensDetails] = None - image_tokens: Optional[int] = 0 - - class Config: - """Pydantic configuration for TokenDetails.""" - - extra = "allow" - - -class Usage(BaseModel): - """Token usage statistics for a response. - - Parameters: - total_tokens: Total number of tokens used. - input_tokens: Number of input tokens used. - output_tokens: Number of output tokens used. - input_token_details: Detailed breakdown of input token usage. - output_token_details: Detailed breakdown of output token usage. - """ - - total_tokens: int - input_tokens: int - output_tokens: int - input_token_details: TokenDetails - output_token_details: TokenDetails - - -class Response(BaseModel): - """A complete assistant response. - - Parameters: - id: Unique identifier for the response. - object: Object type, always "realtime.response". - status: Current status of the response. - status_details: Additional status information. - output: List of conversation items in the response. - conversation_id: Which conversation the response is added to. - output_modalities: The set of modalities the model used to respond. - max_output_tokens: Maximum number of output tokens used. - audio: Audio configuration for the response. - usage: Token usage statistics for the response. - voice: The voice the model used to respond. - temperature: Sampling temperature used for the response. - output_audio_format: The format of output audio. - """ - - id: str - object: Literal["realtime.response"] - status: Literal["completed", "in_progress", "incomplete", "cancelled", "failed"] - status_details: Any - output: List[ConversationItem] - output_modalities: Optional[List[Literal["text", "audio"]]] = None - max_output_tokens: Optional[Union[int, Literal["inf"]]] = None - audio: Optional[AudioConfiguration] = None - usage: Optional[Usage] = None - voice: Optional[str] = None - temperature: Optional[float] = None - output_audio_format: Optional[str] = None - - -_server_event_types = { - "error": ErrorEvent, - "session.created": SessionCreatedEvent, - "session.updated": SessionUpdatedEvent, - "conversation.created": ConversationCreated, - "input_audio_buffer.committed": InputAudioBufferCommitted, - "input_audio_buffer.cleared": InputAudioBufferCleared, - "input_audio_buffer.speech_started": InputAudioBufferSpeechStarted, - "input_audio_buffer.speech_stopped": InputAudioBufferSpeechStopped, - "conversation.item.added": ConversationItemAdded, - "conversation.item.done": ConversationItemDone, - "conversation.item.input_audio_transcription.delta": ConversationItemInputAudioTranscriptionDelta, - "conversation.item.input_audio_transcription.completed": ConversationItemInputAudioTranscriptionCompleted, - "conversation.item.input_audio_transcription.failed": ConversationItemInputAudioTranscriptionFailed, - "conversation.item.truncated": ConversationItemTruncated, - "conversation.item.deleted": ConversationItemDeleted, - "conversation.item.retrieved": ConversationItemRetrieved, - "response.created": ResponseCreated, - "response.done": ResponseDone, - "response.output_item.added": ResponseOutputItemAdded, - "response.output_item.done": ResponseOutputItemDone, - "response.content_part.added": ResponseContentPartAdded, - "response.content_part.done": ResponseContentPartDone, - "response.output_text.delta": ResponseTextDelta, - "response.output_text.done": ResponseTextDone, - "response.output_audio_transcript.delta": ResponseAudioTranscriptDelta, - "response.output_audio_transcript.done": ResponseAudioTranscriptDone, - "response.output_audio.delta": ResponseAudioDelta, - "response.output_audio.done": ResponseAudioDone, - "response.function_call_arguments.delta": ResponseFunctionCallArgumentsDelta, - "response.function_call_arguments.done": ResponseFunctionCallArgumentsDone, - "rate_limits.updated": RateLimitsUpdated, -} - - -def parse_server_event(str): - """Parse a server event from JSON string. - - Args: - str: JSON string containing the server event. - - Returns: - Parsed server event object of the appropriate type. - - Raises: - Exception: If the event type is unimplemented or parsing fails. - """ - try: - event = json.loads(str) - event_type = event["type"] - if event_type not in _server_event_types: - raise Exception(f"Unimplemented server event type: {event_type}") - return _server_event_types[event_type].model_validate(event) - except Exception as e: - raise Exception(f"{e} \n\n{str}") +import warnings + +from pipecat.services.openai.realtime.events import * + +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.events are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.events instead.", + DeprecationWarning, + stacklevel=2, + ) diff --git a/src/pipecat/services/openai_realtime/frames.py b/src/pipecat/services/openai_realtime/frames.py index 290e025f9..e7e4d7d9f 100644 --- a/src/pipecat/services/openai_realtime/frames.py +++ b/src/pipecat/services/openai_realtime/frames.py @@ -1,37 +1,21 @@ # -# Copyright (c) 2024–2025, Daily +# Copyright (c) 2025, Daily # # SPDX-License-Identifier: BSD 2-Clause License # """Custom frame types for OpenAI Realtime API integration.""" -from dataclasses import dataclass -from typing import TYPE_CHECKING +import warnings -from pipecat.frames.frames import DataFrame, FunctionCallResultFrame +from pipecat.services.openai.realtime.frames import * -if TYPE_CHECKING: - from pipecat.services.openai_realtime.context import OpenAIRealtimeLLMContext - - -@dataclass -class RealtimeMessagesUpdateFrame(DataFrame): - """Frame indicating that the realtime context messages have been updated. - - Parameters: - context: The updated OpenAI realtime LLM context. - """ - - context: "OpenAIRealtimeLLMContext" - - -@dataclass -class RealtimeFunctionCallResultFrame(DataFrame): - """Frame containing function call results for the realtime service. - - Parameters: - result_frame: The function call result frame to send to the realtime API. - """ - - result_frame: FunctionCallResultFrame +with warnings.catch_warnings(): + warnings.simplefilter("always") + warnings.warn( + "Types in pipecat.services.openai_realtime.frames are deprecated. " + "Please use the equivalent types from " + "pipecat.services.openai.realtime.frames instead.", + DeprecationWarning, + stacklevel=2, + )