From 2aadac7a4dfe4f39c2d71ec6d6b31aa0c2078813 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 8 Jan 2026 13:17:07 -0500 Subject: [PATCH] Update OpenAIRealtime image to video to align with GeminiLive --- changelog/3360.added.md | 12 ++-- .../19c-openai-realtime-live-video.py | 5 +- src/pipecat/services/openai/realtime/llm.py | 63 ++++++++++--------- uv.lock | 8 +-- 4 files changed, 45 insertions(+), 43 deletions(-) diff --git a/changelog/3360.added.md b/changelog/3360.added.md index 5f5d92941..c9f22e823 100644 --- a/changelog/3360.added.md +++ b/changelog/3360.added.md @@ -1,6 +1,8 @@ - Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`: - - New `start_image_paused` parameter to control initial image input state - - New `image_detail` parameter to set image processing quality ("auto", "low", or "high") - - `set_image_input_paused()` method to pause/resume image input at runtime - - `set_image_detail()` method to adjust image quality dynamically - - Automatic rate limiting (1 image per second) to prevent API overload + - New `start_video_paused` parameter to control initial video input state + - New `video_frame_detail` parameter to set image processing quality ("auto", + "low", or "high"). This corresponds to OpenAI Realtime's `image_detail` + parameter. + - `set_video_input_paused()` method to pause/resume video input at runtime + - `set_video_frame_detail()` method to adjust video frame quality dynamically + - Automatic rate limiting (1 frame per second) to prevent API overload diff --git a/examples/foundational/19c-openai-realtime-live-video.py b/examples/foundational/19c-openai-realtime-live-video.py index 5c1a90777..af48b2355 100644 --- a/examples/foundational/19c-openai-realtime-live-video.py +++ b/examples/foundational/19c-openai-realtime-live-video.py @@ -5,7 +5,6 @@ # -import asyncio import os from dotenv import load_dotenv @@ -131,8 +130,8 @@ Remember, your responses should be short. Just one or two sentences, usually. Re async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - await maybe_capture_participant_camera(transport, client, framerate=1) - await maybe_capture_participant_screen(transport, client, framerate=1) + await maybe_capture_participant_camera(transport, client, framerate=0.5) + await maybe_capture_participant_screen(transport, client, framerate=0.5) await task.queue_frames([LLMRunFrame()]) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 9f413b33a..598d4f510 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -109,8 +109,8 @@ class OpenAIRealtimeLLMService(LLMService): base_url: str = "wss://api.openai.com/v1/realtime", session_properties: Optional[events.SessionProperties] = None, start_audio_paused: bool = False, - start_image_paused: bool = False, - image_detail: str = "auto", + start_video_paused: bool = False, + video_frame_detail: str = "auto", send_transcription_frames: Optional[bool] = None, **kwargs, ): @@ -127,8 +127,9 @@ class OpenAIRealtimeLLMService(LLMService): These are session-level settings that can be updated during the session (except for voice and model). If None, uses default SessionProperties. start_audio_paused: Whether to start with audio input paused. Defaults to False. - start_image_paused: Whether to start with image input paused. Defaults to False. - image_detail: Detail level for image processing. Can be "auto", "low", or "high". + start_video_paused: Whether to start with video input paused. Defaults to False. + video_frame_detail: Detail level for video processing. Can be "auto", "low", or "high". + This sets the image_detail parameter in the OpenAI Realtime API. "auto" lets the model decide, "low" is faster and uses fewer tokens, "high" provides more detail. Defaults to "auto". send_transcription_frames: Whether to emit transcription frames. @@ -165,9 +166,9 @@ class OpenAIRealtimeLLMService(LLMService): session_properties or events.SessionProperties() ) self._audio_input_paused = start_audio_paused - self._image_input_paused = start_image_paused - self._image_detail = image_detail - self._last_image_sent_time = 0 + self._video_input_paused = start_video_paused + self._video_frame_detail = video_frame_detail + self._last_sent_time = 0 self._websocket = None self._receive_task = None self._context: LLMContext = None @@ -205,24 +206,24 @@ class OpenAIRealtimeLLMService(LLMService): """ self._audio_input_paused = paused - def set_image_input_paused(self, paused: bool): - """Set whether image input is paused. + def set_video_input_paused(self, paused: bool): + """Set whether video input is paused. Args: - paused: True to pause image input, False to resume. + paused: True to pause video input, False to resume. """ - self._image_input_paused = paused + self._video_input_paused = paused - def set_image_detail(self, detail: str): - """Set the detail level for image processing. + def set_video_frame_detail(self, detail: str): + """Set the detail level for video processing. Args: detail: Detail level - "auto", "low", or "high". """ if detail not in ["auto", "low", "high"]: - logger.warning(f"Invalid image detail '{detail}', must be 'auto', 'low', or 'high'") + logger.warning(f"Invalid video detail '{detail}', must be 'auto', 'low', or 'high'") return - self._image_detail = detail + self._video_frame_detail = detail def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" @@ -411,8 +412,8 @@ class OpenAIRealtimeLLMService(LLMService): if not self._audio_input_paused: await self._send_user_audio(frame) elif isinstance(frame, InputImageRawFrame): - if not self._image_input_paused: - await self._send_user_image(frame) + if not self._video_input_paused: + await self._send_user_video(frame) elif isinstance(frame, InterruptionFrame): await self._handle_interruption() elif isinstance(frame, UserStartedSpeakingFrame): @@ -881,39 +882,39 @@ class OpenAIRealtimeLLMService(LLMService): payload = base64.b64encode(frame.audio).decode("utf-8") await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) - async def _send_user_image(self, frame: InputImageRawFrame): - """Send user image frame to OpenAI Realtime API. + async def _send_user_video(self, frame: InputImageRawFrame): + """Send user video frame to OpenAI Realtime API. Args: - frame: The input image frame to send. + frame: The InputImageRawFrame to send. """ - if self._image_input_paused or self._disconnecting or not self._websocket: + if self._video_input_paused or self._disconnecting or not self._websocket: return now = time.time() - if now - self._last_image_sent_time < 1: + if now - self._last_sent_time < 1: return # Ignore if less than 1 second has passed - self._last_image_sent_time = now # Update last sent time - logger.trace(f"Sending image frame to OpenAI Realtime: {frame}") + self._last_sent_time = now # Update last sent time + logger.trace(f"Sending video frame to OpenAI Realtime: {frame}") - # Convert image to JPEG format and encode as base64 + # Convert video frame to JPEG format and encode as base64 buffer = io.BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") - image_data = base64.b64encode(buffer.getvalue()).decode("utf-8") + data = base64.b64encode(buffer.getvalue()).decode("utf-8") - # Create data URI for the image - image_url = f"data:image/jpeg;base64,{image_data}" + # Create data URI for the video frame + data_uri = f"data:image/jpeg;base64,{data}" - # Create a conversation item with the image + # Create a conversation item with the video frame item = events.ConversationItem( type="message", role="user", content=[ events.ItemContent( type="input_image", - image_url=image_url, - detail=self._image_detail, + image_url=data_uri, + detail=self._video_frame_detail, ) ], ) diff --git a/uv.lock b/uv.lock index 7dd5e0562..51a0a2ec1 100644 --- a/uv.lock +++ b/uv.lock @@ -4208,7 +4208,7 @@ requires-dist = [ { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, - { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.4" }, + { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.6" }, { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, @@ -5948,16 +5948,16 @@ wheels = [ [[package]] name = "speechmatics-voice" -version = "0.2.4" +version = "0.2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "pydantic" }, { name = "speechmatics-rt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b2/f9/9d81e4abe9ae1c8745372eaf43523213b0333e9721699fb0f3d3bff6c17e/speechmatics_voice-0.2.4.tar.gz", hash = "sha256:e3b5c7a8c24fa7d555b80a72ab181797665c74944400468ca5fb7e54b5f9eae6", size = 60852, upload-time = "2025-12-17T23:22:13.437Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/18/7790718c826be18eadaa7cfc0cc9f229d157f5f3aff628b9fc0f180a7878/speechmatics_voice-0.2.6.tar.gz", hash = "sha256:ae384e8f97862fc6adf38937e1d1d63cd16b64bc49aded8ccad273155634a636", size = 60881, upload-time = "2026-01-08T00:54:41.405Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/a6/401dba9be6be914e57b7814360ba0bece55f24140bb7d5c3dc5f07bcd77f/speechmatics_voice-0.2.4-py3-none-any.whl", hash = "sha256:71d0f5272c2db1221422ab19b6c898ea7b38f9fb7f523904f54a4d8c3e4cef12", size = 57056, upload-time = "2025-12-17T23:22:11.837Z" }, + { url = "https://files.pythonhosted.org/packages/e2/cc/ae6dc3d5638a3fc86c4537af1fb394dee3c4a2a5e9dbebf9fb83a8052939/speechmatics_voice-0.2.6-py3-none-any.whl", hash = "sha256:15d61cb02d7fe492f966cc28ddb0ada199fdd12543b9a61cb8757c7bf25b7a94", size = 57103, upload-time = "2026-01-08T00:54:39.92Z" }, ] [package.optional-dependencies]