wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update OpenAIRealtime image to video to align with GeminiLive

Browse Source
This commit is contained in:
Mark Backman
2026-01-08 13:17:07 -05:00
parent b90a34228f
commit 2aadac7a4d
4 changed files with 45 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
changelog/3360.added.md
View File

@@ -1,6 +1,8 @@
- Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`:
- New `start_image_paused` parameter to control initial image input state
- New `image_detail` parameter to set image processing quality ("auto", "low", or "high")
- `set_image_input_paused()` method to pause/resume image input at runtime
- `set_image_detail()` method to adjust image quality dynamically
- Automatic rate limiting (1 image per second) to prevent API overload
- New `start_video_paused` parameter to control initial video input state
- New `video_frame_detail` parameter to set image processing quality ("auto",
"low", or "high"). This corresponds to OpenAI Realtime's `image_detail`
parameter.
- `set_video_input_paused()` method to pause/resume video input at runtime
- `set_video_frame_detail()` method to adjust video frame quality dynamically
- Automatic rate limiting (1 frame per second) to prevent API overload

5
examples/foundational/19c-openai-realtime-live-video.py
View File

@@ -5,7 +5,6 @@
#
import asyncio
import os
from dotenv import load_dotenv
@@ -131,8 +130,8 @@ Remember, your responses should be short. Just one or two sentences, usually. Re
async def on_client_connected(transport, client):
logger.info(f"Client connected: {client}")
await maybe_capture_participant_camera(transport, client, framerate=1)
await maybe_capture_participant_screen(transport, client, framerate=1)
await maybe_capture_participant_camera(transport, client, framerate=0.5)
await maybe_capture_participant_screen(transport, client, framerate=0.5)
await task.queue_frames([LLMRunFrame()])

63
src/pipecat/services/openai/realtime/llm.py
View File

@@ -109,8 +109,8 @@ class OpenAIRealtimeLLMService(LLMService):
base_url: str = "wss://api.openai.com/v1/realtime",
session_properties: Optional[events.SessionProperties] = None,
start_audio_paused: bool = False,
start_image_paused: bool = False,
image_detail: str = "auto",
start_video_paused: bool = False,
video_frame_detail: str = "auto",
send_transcription_frames: Optional[bool] = None,
**kwargs,
):
@@ -127,8 +127,9 @@ class OpenAIRealtimeLLMService(LLMService):
These are session-level settings that can be updated during the session
(except for voice and model). If None, uses default SessionProperties.
start_audio_paused: Whether to start with audio input paused. Defaults to False.
start_image_paused: Whether to start with image input paused. Defaults to False.
image_detail: Detail level for image processing. Can be "auto", "low", or "high".
start_video_paused: Whether to start with video input paused. Defaults to False.
video_frame_detail: Detail level for video processing. Can be "auto", "low", or "high".
This sets the image_detail parameter in the OpenAI Realtime API.
"auto" lets the model decide, "low" is faster and uses fewer tokens,
"high" provides more detail. Defaults to "auto".
send_transcription_frames: Whether to emit transcription frames.
@@ -165,9 +166,9 @@ class OpenAIRealtimeLLMService(LLMService):
session_properties or events.SessionProperties()
)
self._audio_input_paused = start_audio_paused
self._image_input_paused = start_image_paused
self._image_detail = image_detail
self._last_image_sent_time = 0
self._video_input_paused = start_video_paused
self._video_frame_detail = video_frame_detail
self._last_sent_time = 0
self._websocket = None
self._receive_task = None
self._context: LLMContext = None
@@ -205,24 +206,24 @@ class OpenAIRealtimeLLMService(LLMService):
"""
self._audio_input_paused = paused
def set_image_input_paused(self, paused: bool):
"""Set whether image input is paused.
def set_video_input_paused(self, paused: bool):
"""Set whether video input is paused.
Args:
paused: True to pause image input, False to resume.
paused: True to pause video input, False to resume.
"""
self._image_input_paused = paused
self._video_input_paused = paused
def set_image_detail(self, detail: str):
"""Set the detail level for image processing.
def set_video_frame_detail(self, detail: str):
"""Set the detail level for video processing.
Args:
detail: Detail level - "auto", "low", or "high".
"""
if detail not in ["auto", "low", "high"]:
logger.warning(f"Invalid image detail '{detail}', must be 'auto', 'low', or 'high'")
logger.warning(f"Invalid video detail '{detail}', must be 'auto', 'low', or 'high'")
return
self._image_detail = detail
self._video_frame_detail = detail
def _is_modality_enabled(self, modality: str) -> bool:
"""Check if a specific modality is enabled, "text" or "audio"."""
@@ -411,8 +412,8 @@ class OpenAIRealtimeLLMService(LLMService):
if not self._audio_input_paused:
await self._send_user_audio(frame)
elif isinstance(frame, InputImageRawFrame):
if not self._image_input_paused:
await self._send_user_image(frame)
if not self._video_input_paused:
await self._send_user_video(frame)
elif isinstance(frame, InterruptionFrame):
await self._handle_interruption()
elif isinstance(frame, UserStartedSpeakingFrame):
@@ -881,39 +882,39 @@ class OpenAIRealtimeLLMService(LLMService):
payload = base64.b64encode(frame.audio).decode("utf-8")
await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload))
async def _send_user_image(self, frame: InputImageRawFrame):
"""Send user image frame to OpenAI Realtime API.
async def _send_user_video(self, frame: InputImageRawFrame):
"""Send user video frame to OpenAI Realtime API.
Args:
frame: The input image frame to send.
frame: The InputImageRawFrame to send.
"""
if self._image_input_paused or self._disconnecting or not self._websocket:
if self._video_input_paused or self._disconnecting or not self._websocket:
return
now = time.time()
if now - self._last_image_sent_time < 1:
if now - self._last_sent_time < 1:
return # Ignore if less than 1 second has passed
self._last_image_sent_time = now # Update last sent time
logger.trace(f"Sending image frame to OpenAI Realtime: {frame}")
self._last_sent_time = now # Update last sent time
logger.trace(f"Sending video frame to OpenAI Realtime: {frame}")
# Convert image to JPEG format and encode as base64
# Convert video frame to JPEG format and encode as base64
buffer = io.BytesIO()
Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG")
image_data = base64.b64encode(buffer.getvalue()).decode("utf-8")
data = base64.b64encode(buffer.getvalue()).decode("utf-8")
# Create data URI for the image
image_url = f"data:image/jpeg;base64,{image_data}"
# Create data URI for the video frame
data_uri = f"data:image/jpeg;base64,{data}"
# Create a conversation item with the image
# Create a conversation item with the video frame
item = events.ConversationItem(
type="message",
role="user",
content=[
events.ItemContent(
type="input_image",
image_url=image_url,
detail=self._image_detail,
image_url=data_uri,
detail=self._video_frame_detail,
)
],
)

8
uv.lock generated
View File

@@ -4208,7 +4208,7 @@ requires-dist = [
{ name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" },
{ name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" },
{ name = "soxr", specifier = "~=0.5.0" },
{ name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.4" },
{ name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.6" },
{ name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" },
{ name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" },
{ name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" },
@@ -5948,16 +5948,16 @@ wheels = [
[[package]]
name = "speechmatics-voice"
version = "0.2.4"
version = "0.2.6"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy" },
{ name = "pydantic" },
{ name = "speechmatics-rt" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b2/f9/9d81e4abe9ae1c8745372eaf43523213b0333e9721699fb0f3d3bff6c17e/speechmatics_voice-0.2.4.tar.gz", hash = "sha256:e3b5c7a8c24fa7d555b80a72ab181797665c74944400468ca5fb7e54b5f9eae6", size = 60852, upload-time = "2025-12-17T23:22:13.437Z" }
sdist = { url = "https://files.pythonhosted.org/packages/d2/18/7790718c826be18eadaa7cfc0cc9f229d157f5f3aff628b9fc0f180a7878/speechmatics_voice-0.2.6.tar.gz", hash = "sha256:ae384e8f97862fc6adf38937e1d1d63cd16b64bc49aded8ccad273155634a636", size = 60881, upload-time = "2026-01-08T00:54:41.405Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/69/a6/401dba9be6be914e57b7814360ba0bece55f24140bb7d5c3dc5f07bcd77f/speechmatics_voice-0.2.4-py3-none-any.whl", hash = "sha256:71d0f5272c2db1221422ab19b6c898ea7b38f9fb7f523904f54a4d8c3e4cef12", size = 57056, upload-time = "2025-12-17T23:22:11.837Z" },
{ url = "https://files.pythonhosted.org/packages/e2/cc/ae6dc3d5638a3fc86c4537af1fb394dee3c4a2a5e9dbebf9fb83a8052939/speechmatics_voice-0.2.6-py3-none-any.whl", hash = "sha256:15d61cb02d7fe492f966cc28ddb0ada199fdd12543b9a61cb8757c7bf25b7a94", size = 57103, upload-time = "2026-01-08T00:54:39.92Z" },
]
[package.optional-dependencies]