wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OpenAIRealtime image to video to align with GeminiLive

Browse Source
This commit is contained in:
Mark Backman
2026-01-08 13:17:07 -05:00
parent b90a34228f
commit 2aadac7a4d
4 changed files with 45 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
changelog/3360.added.md
View File

@@ -1,6 +1,8 @@
- Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`: - Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`:
- New `start_image_paused` parameter to control initial image input state - New `start_video_paused` parameter to control initial video input state
- New `image_detail` parameter to set image processing quality ("auto", "low", or "high") - New `video_frame_detail` parameter to set image processing quality ("auto",
- `set_image_input_paused()` method to pause/resume image input at runtime "low", or "high"). This corresponds to OpenAI Realtime's `image_detail`
- `set_image_detail()` method to adjust image quality dynamically parameter.
- Automatic rate limiting (1 image per second) to prevent API overload - `set_video_input_paused()` method to pause/resume video input at runtime
- `set_video_frame_detail()` method to adjust video frame quality dynamically
- Automatic rate limiting (1 frame per second) to prevent API overload

5
examples/foundational/19c-openai-realtime-live-video.py
View File

@@ -5,7 +5,6 @@
# #
import asyncio
import os import os
from dotenv import load_dotenv from dotenv import load_dotenv
@@ -131,8 +130,8 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
async def on_client_connected(transport, client): async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}") logger.info(f"Client connected: {client}")
await maybe_capture_participant_camera(transport, client, framerate=1) await maybe_capture_participant_camera(transport, client, framerate=0.5)
await maybe_capture_participant_screen(transport, client, framerate=1) await maybe_capture_participant_screen(transport, client, framerate=0.5)
await task.queue_frames([LLMRunFrame()]) await task.queue_frames([LLMRunFrame()])

63
src/pipecat/services/openai/realtime/llm.py
View File

@@ -109,8 +109,8 @@ class OpenAIRealtimeLLMService(LLMService):
base_url: str = "wss://api.openai.com/v1/realtime", base_url: str = "wss://api.openai.com/v1/realtime",
session_properties: Optional[events.SessionProperties] = None, session_properties: Optional[events.SessionProperties] = None,
start_audio_paused: bool = False, start_audio_paused: bool = False,
start_image_paused: bool = False, start_video_paused: bool = False,
image_detail: str = "auto", video_frame_detail: str = "auto",
send_transcription_frames: Optional[bool] = None, send_transcription_frames: Optional[bool] = None,
**kwargs, **kwargs,
): ):
@@ -127,8 +127,9 @@ class OpenAIRealtimeLLMService(LLMService):
These are session-level settings that can be updated during the session These are session-level settings that can be updated during the session
(except for voice and model). If None, uses default SessionProperties. (except for voice and model). If None, uses default SessionProperties.
start_audio_paused: Whether to start with audio input paused. Defaults to False. start_audio_paused: Whether to start with audio input paused. Defaults to False.
start_image_paused: Whether to start with image input paused. Defaults to False. start_video_paused: Whether to start with video input paused. Defaults to False.
image_detail: Detail level for image processing. Can be "auto", "low", or "high". video_frame_detail: Detail level for video processing. Can be "auto", "low", or "high".
This sets the image_detail parameter in the OpenAI Realtime API.
"auto" lets the model decide, "low" is faster and uses fewer tokens, "auto" lets the model decide, "low" is faster and uses fewer tokens,
"high" provides more detail. Defaults to "auto". "high" provides more detail. Defaults to "auto".
send_transcription_frames: Whether to emit transcription frames. send_transcription_frames: Whether to emit transcription frames.
@@ -165,9 +166,9 @@ class OpenAIRealtimeLLMService(LLMService):
session_properties or events.SessionProperties() session_properties or events.SessionProperties()
) )
self._audio_input_paused = start_audio_paused self._audio_input_paused = start_audio_paused
self._image_input_paused = start_image_paused self._video_input_paused = start_video_paused
self._image_detail = image_detail self._video_frame_detail = video_frame_detail
self._last_image_sent_time = 0 self._last_sent_time = 0
self._websocket = None self._websocket = None
self._receive_task = None self._receive_task = None
self._context: LLMContext = None self._context: LLMContext = None
@@ -205,24 +206,24 @@ class OpenAIRealtimeLLMService(LLMService):
""" """
self._audio_input_paused = paused self._audio_input_paused = paused
def set_image_input_paused(self, paused: bool): def set_video_input_paused(self, paused: bool):
"""Set whether image input is paused. """Set whether video input is paused.
Args: Args:
paused: True to pause image input, False to resume. paused: True to pause video input, False to resume.
""" """
self._image_input_paused = paused self._video_input_paused = paused
def set_image_detail(self, detail: str): def set_video_frame_detail(self, detail: str):
"""Set the detail level for image processing. """Set the detail level for video processing.
Args: Args:
detail: Detail level - "auto", "low", or "high". detail: Detail level - "auto", "low", or "high".
""" """
if detail not in ["auto", "low", "high"]: if detail not in ["auto", "low", "high"]:
logger.warning(f"Invalid image detail '{detail}', must be 'auto', 'low', or 'high'") logger.warning(f"Invalid video detail '{detail}', must be 'auto', 'low', or 'high'")
return return
self._image_detail = detail self._video_frame_detail = detail
def _is_modality_enabled(self, modality: str) -> bool: def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio".""" """Check if a specific modality is enabled, "text" or "audio"."""
@@ -411,8 +412,8 @@ class OpenAIRealtimeLLMService(LLMService):
if not self._audio_input_paused: if not self._audio_input_paused:
await self._send_user_audio(frame) await self._send_user_audio(frame)
elif isinstance(frame, InputImageRawFrame): elif isinstance(frame, InputImageRawFrame):
if not self._image_input_paused: if not self._video_input_paused:
await self._send_user_image(frame) await self._send_user_video(frame)
elif isinstance(frame, InterruptionFrame): elif isinstance(frame, InterruptionFrame):
await self._handle_interruption() await self._handle_interruption()
elif isinstance(frame, UserStartedSpeakingFrame): elif isinstance(frame, UserStartedSpeakingFrame):
@@ -881,39 +882,39 @@ class OpenAIRealtimeLLMService(LLMService):
payload = base64.b64encode(frame.audio).decode("utf-8") payload = base64.b64encode(frame.audio).decode("utf-8")
await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload))
async def _send_user_image(self, frame: InputImageRawFrame): async def _send_user_video(self, frame: InputImageRawFrame):
"""Send user image frame to OpenAI Realtime API. """Send user video frame to OpenAI Realtime API.
Args: Args:
frame: The input image frame to send. frame: The InputImageRawFrame to send.
""" """
if self._image_input_paused or self._disconnecting or not self._websocket: if self._video_input_paused or self._disconnecting or not self._websocket:
return return
now = time.time() now = time.time()
if now - self._last_image_sent_time < 1: if now - self._last_sent_time < 1:
return # Ignore if less than 1 second has passed return # Ignore if less than 1 second has passed
self._last_image_sent_time = now # Update last sent time self._last_sent_time = now # Update last sent time
logger.trace(f"Sending image frame to OpenAI Realtime: {frame}") logger.trace(f"Sending video frame to OpenAI Realtime: {frame}")
# Convert image to JPEG format and encode as base64 # Convert video frame to JPEG format and encode as base64
buffer = io.BytesIO() buffer = io.BytesIO()
Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
image_data = base64.b64encode(buffer.getvalue()).decode("utf-8") data = base64.b64encode(buffer.getvalue()).decode("utf-8")
# Create data URI for the image # Create data URI for the video frame
image_url = f"data:image/jpeg;base64,{image_data}" data_uri = f"data:image/jpeg;base64,{data}"
# Create a conversation item with the image # Create a conversation item with the video frame
item = events.ConversationItem( item = events.ConversationItem(
type="message", type="message",
role="user", role="user",
content=[ content=[
events.ItemContent( events.ItemContent(
type="input_image", type="input_image",
image_url=image_url, image_url=data_uri,
detail=self._image_detail, detail=self._video_frame_detail,
) )
], ],
) )

8
uv.lock generated
View File

@@ -4208,7 +4208,7 @@ requires-dist = [
{ name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" },
{ name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" },
{ name = "soxr", specifier = "~=0.5.0" }, { name = "soxr", specifier = "~=0.5.0" },
{ name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.4" }, { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.6" },
{ name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" },
{ name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" },
{ name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" },
@@ -5948,16 +5948,16 @@ wheels = [
[[package]] [[package]]
name = "speechmatics-voice" name = "speechmatics-voice"
version = "0.2.4" version = "0.2.6"
source = { registry = "https://pypi.org/simple" } source = { registry = "https://pypi.org/simple" }
dependencies = [ dependencies = [
{ name = "numpy" }, { name = "numpy" },
{ name = "pydantic" }, { name = "pydantic" },
{ name = "speechmatics-rt" }, { name = "speechmatics-rt" },
] ]
sdist = { url = "https://files.pythonhosted.org/packages/b2/f9/9d81e4abe9ae1c8745372eaf43523213b0333e9721699fb0f3d3bff6c17e/speechmatics_voice-0.2.4.tar.gz", hash = "sha256:e3b5c7a8c24fa7d555b80a72ab181797665c74944400468ca5fb7e54b5f9eae6", size = 60852, upload-time = "2025-12-17T23:22:13.437Z" } sdist = { url = "https://files.pythonhosted.org/packages/d2/18/7790718c826be18eadaa7cfc0cc9f229d157f5f3aff628b9fc0f180a7878/speechmatics_voice-0.2.6.tar.gz", hash = "sha256:ae384e8f97862fc6adf38937e1d1d63cd16b64bc49aded8ccad273155634a636", size = 60881, upload-time = "2026-01-08T00:54:41.405Z" }
wheels = [ wheels = [
{ url = "https://files.pythonhosted.org/packages/69/a6/401dba9be6be914e57b7814360ba0bece55f24140bb7d5c3dc5f07bcd77f/speechmatics_voice-0.2.4-py3-none-any.whl", hash = "sha256:71d0f5272c2db1221422ab19b6c898ea7b38f9fb7f523904f54a4d8c3e4cef12", size = 57056, upload-time = "2025-12-17T23:22:11.837Z" }, { url = "https://files.pythonhosted.org/packages/e2/cc/ae6dc3d5638a3fc86c4537af1fb394dee3c4a2a5e9dbebf9fb83a8052939/speechmatics_voice-0.2.6-py3-none-any.whl", hash = "sha256:15d61cb02d7fe492f966cc28ddb0ada199fdd12543b9a61cb8757c7bf25b7a94", size = 57103, upload-time = "2026-01-08T00:54:39.92Z" },
] ]
[package.optional-dependencies] [package.optional-dependencies]