wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OpenAIRealtime image to video to align with GeminiLive

Browse Source
This commit is contained in:
Mark Backman
2026-01-08 13:17:07 -05:00
parent b90a34228f
commit 2aadac7a4d
4 changed files with 45 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
src/pipecat/services/openai/realtime/llm.py
View File

@@ -109,8 +109,8 @@ class OpenAIRealtimeLLMService(LLMService):
base_url: str = "wss://api.openai.com/v1/realtime",
session_properties: Optional[events.SessionProperties] = None,
start_audio_paused: bool = False,
start_image_paused: bool = False,
image_detail: str = "auto",
start_video_paused: bool = False,
video_frame_detail: str = "auto",
send_transcription_frames: Optional[bool] = None,
**kwargs,
):
@@ -127,8 +127,9 @@ class OpenAIRealtimeLLMService(LLMService):
These are session-level settings that can be updated during the session
(except for voice and model). If None, uses default SessionProperties.
start_audio_paused: Whether to start with audio input paused. Defaults to False.
start_image_paused: Whether to start with image input paused. Defaults to False.
image_detail: Detail level for image processing. Can be "auto", "low", or "high".
start_video_paused: Whether to start with video input paused. Defaults to False.
video_frame_detail: Detail level for video processing. Can be "auto", "low", or "high".
This sets the image_detail parameter in the OpenAI Realtime API.
"auto" lets the model decide, "low" is faster and uses fewer tokens,
"high" provides more detail. Defaults to "auto".
send_transcription_frames: Whether to emit transcription frames.
@@ -165,9 +166,9 @@ class OpenAIRealtimeLLMService(LLMService):
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._image_input_paused = start_image_paused
self._image_detail = image_detail
self._last_image_sent_time = 0
self._video_input_paused = start_video_paused
self._video_frame_detail = video_frame_detail
self._last_sent_time = 0
self._websocket = None
self._receive_task = None
self._context: LLMContext = None
@@ -205,24 +206,24 @@ class OpenAIRealtimeLLMService(LLMService):
"""
self._audio_input_paused = paused
def set_image_input_paused(self, paused: bool):
"""Set whether image input is paused.
def set_video_input_paused(self, paused: bool):
"""Set whether video input is paused.
Args:
paused: True to pause image input, False to resume.
paused: True to pause video input, False to resume.
"""
self._image_input_paused = paused
self._video_input_paused = paused
def set_image_detail(self, detail: str):
"""Set the detail level for image processing.
def set_video_frame_detail(self, detail: str):
"""Set the detail level for video processing.
Args:
detail: Detail level - "auto", "low", or "high".
"""
if detail not in ["auto", "low", "high"]:
logger.warning(f"Invalid image detail '{detail}', must be 'auto', 'low', or 'high'")
logger.warning(f"Invalid video detail '{detail}', must be 'auto', 'low', or 'high'")
return
self._image_detail = detail
self._video_frame_detail = detail
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
@@ -411,8 +412,8 @@ class OpenAIRealtimeLLMService(LLMService):
if not self._audio_input_paused:
await self._send_user_audio(frame)
elif isinstance(frame, InputImageRawFrame):
if not self._image_input_paused:
await self._send_user_image(frame)
if not self._video_input_paused:
await self._send_user_video(frame)
elif isinstance(frame, InterruptionFrame):
await self._handle_interruption()
elif isinstance(frame, UserStartedSpeakingFrame):
@@ -881,39 +882,39 @@ class OpenAIRealtimeLLMService(LLMService):
payload = base64.b64encode(frame.audio).decode("utf-8")
await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload))
async def _send_user_image(self, frame: InputImageRawFrame):
"""Send user image frame to OpenAI Realtime API.
async def _send_user_video(self, frame: InputImageRawFrame):
"""Send user video frame to OpenAI Realtime API.
Args:
frame: The input image frame to send.
frame: The InputImageRawFrame to send.
"""
if self._image_input_paused or self._disconnecting or not self._websocket:
if self._video_input_paused or self._disconnecting or not self._websocket:
return
now = time.time()
if now - self._last_image_sent_time < 1:
if now - self._last_sent_time < 1:
return # Ignore if less than 1 second has passed
self._last_image_sent_time = now # Update last sent time
logger.trace(f"Sending image frame to OpenAI Realtime: {frame}")
self._last_sent_time = now # Update last sent time
logger.trace(f"Sending video frame to OpenAI Realtime: {frame}")
# Convert image to JPEG format and encode as base64
# Convert video frame to JPEG format and encode as base64
buffer = io.BytesIO()
Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
image_data = base64.b64encode(buffer.getvalue()).decode("utf-8")
data = base64.b64encode(buffer.getvalue()).decode("utf-8")
# Create data URI for the image
image_url = f"data:image/jpeg;base64,{image_data}"
# Create data URI for the video frame
data_uri = f"data:image/jpeg;base64,{data}"
# Create a conversation item with the image
# Create a conversation item with the video frame
item = events.ConversationItem(
type="message",
role="user",
content=[
events.ItemContent(
type="input_image",
image_url=image_url,
detail=self._image_detail,
image_url=data_uri,
detail=self._video_frame_detail,
)
],
)