From 7ae9eebc344301a112b97e309aae66ae341164df Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 6 Jan 2026 14:11:46 -0500 Subject: [PATCH 1/6] Add image input support for OpenAI Realtime --- changelog/3360.added.md | 6 ++ .../services/openai/realtime/events.py | 10 ++- src/pipecat/services/openai/realtime/llm.py | 77 +++++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 changelog/3360.added.md diff --git a/changelog/3360.added.md b/changelog/3360.added.md new file mode 100644 index 000000000..5f5d92941 --- /dev/null +++ b/changelog/3360.added.md @@ -0,0 +1,6 @@ +- Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`: + - New `start_image_paused` parameter to control initial image input state + - New `image_detail` parameter to set image processing quality ("auto", "low", or "high") + - `set_image_input_paused()` method to pause/resume image input at runtime + - `set_image_detail()` method to adjust image quality dynamically + - Automatic rate limiting (1 image per second) to prevent API overload diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py index 09d1e8ae4..c0ef5b890 100644 --- a/src/pipecat/services/openai/realtime/events.py +++ b/src/pipecat/services/openai/realtime/events.py @@ -217,16 +217,22 @@ class ItemContent(BaseModel): """Content within a conversation item. Parameters: - type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). + type: Content type (text, audio, input_text, input_audio, input_image, output_text, or output_audio). text: Text content for text-based items. audio: Base64-encoded audio data for audio items. transcript: Transcribed text for audio items. + image_url: Base64-encoded image data as a data URI for input_image items. + detail: Detail level for image processing ("auto", "low", or "high"). """ - type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] + type: Literal[ + "text", "audio", "input_text", "input_audio", "input_image", "output_text", "output_audio" + ] text: Optional[str] = None audio: Optional[str] = None # base64-encoded audio transcript: Optional[str] = None + image_url: Optional[str] = None # base64-encoded image as data URI + detail: Optional[Literal["auto", "low", "high"]] = None class ConversationItem(BaseModel): diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 5cb50e16d..21c8e58e8 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -7,12 +7,14 @@ """OpenAI Realtime LLM service implementation with WebSocket support.""" import base64 +import io import json import time from dataclasses import dataclass from typing import Optional from loguru import logger +from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.open_ai_realtime_adapter import ( @@ -25,6 +27,7 @@ from pipecat.frames.frames import ( EndFrame, Frame, InputAudioRawFrame, + InputImageRawFrame, InterimTranscriptionFrame, InterruptionFrame, LLMContextFrame, @@ -106,6 +109,8 @@ class OpenAIRealtimeLLMService(LLMService): base_url: str = "wss://api.openai.com/v1/realtime", session_properties: Optional[events.SessionProperties] = None, start_audio_paused: bool = False, + start_image_paused: bool = False, + image_detail: str = "auto", send_transcription_frames: Optional[bool] = None, **kwargs, ): @@ -122,6 +127,10 @@ class OpenAIRealtimeLLMService(LLMService): These are session-level settings that can be updated during the session (except for voice and model). If None, uses default SessionProperties. start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_image_paused: Whether to start with image input paused. Defaults to False. + image_detail: Detail level for image processing. Can be "auto", "low", or "high". + "auto" lets the model decide, "low" is faster and uses fewer tokens, + "high" provides more detail. Defaults to "auto". send_transcription_frames: Whether to emit transcription frames. .. deprecated:: 0.0.92 @@ -156,6 +165,9 @@ class OpenAIRealtimeLLMService(LLMService): session_properties or events.SessionProperties() ) self._audio_input_paused = start_audio_paused + self._image_input_paused = start_image_paused + self._image_detail = image_detail + self._last_image_sent_time = 0 self._websocket = None self._receive_task = None self._context: LLMContext = None @@ -193,6 +205,25 @@ class OpenAIRealtimeLLMService(LLMService): """ self._audio_input_paused = paused + def set_image_input_paused(self, paused: bool): + """Set whether image input is paused. + + Args: + paused: True to pause image input, False to resume. + """ + self._image_input_paused = paused + + def set_image_detail(self, detail: str): + """Set the detail level for image processing. + + Args: + detail: Detail level - "auto", "low", or "high". + """ + if detail not in ["auto", "low", "high"]: + logger.warning(f"Invalid image detail '{detail}', must be 'auto', 'low', or 'high'") + return + self._image_detail = detail + def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" modalities = self._session_properties.output_modalities or ["audio", "text"] @@ -379,6 +410,9 @@ class OpenAIRealtimeLLMService(LLMService): elif isinstance(frame, InputAudioRawFrame): if not self._audio_input_paused: await self._send_user_audio(frame) + elif isinstance(frame, InputImageRawFrame): + if not self._image_input_paused: + await self._send_user_image(frame) elif isinstance(frame, InterruptionFrame): await self._handle_interruption() elif isinstance(frame, UserStartedSpeakingFrame): @@ -847,6 +881,49 @@ class OpenAIRealtimeLLMService(LLMService): payload = base64.b64encode(frame.audio).decode("utf-8") await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) + async def _send_user_image(self, frame: InputImageRawFrame): + """Send user image frame to OpenAI Realtime API. + + Args: + frame: The input image frame to send. + """ + if self._image_input_paused or self._disconnecting or not self._websocket: + return + + now = time.time() + if now - self._last_image_sent_time < 1: + return # Ignore if less than 1 second has passed + + self._last_image_sent_time = now # Update last sent time + logger.debug(f"Sending image frame to OpenAI Realtime: {frame}") + + # Convert image to JPEG format and encode as base64 + buffer = io.BytesIO() + Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") + image_data = base64.b64encode(buffer.getvalue()).decode("utf-8") + + # Create data URI for the image + image_url = f"data:image/jpeg;base64,{image_data}" + + # Create a conversation item with the image + item = events.ConversationItem( + type="message", + role="user", + content=[ + events.ItemContent( + type="input_image", + image_url=image_url, + detail=self._image_detail, + ) + ], + ) + + # Send the conversation item + try: + await self.send_client_event(events.ConversationItemCreateEvent(item=item)) + except Exception as e: + await self.push_error(error_msg=f"Send error: {e}") + async def _send_tool_result(self, tool_call_id: str, result: str): item = events.ConversationItem( type="function_call_output", From 3a7b489208a3e9ccab971a41da3894bb4f8c130f Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Tue, 6 Jan 2026 14:21:39 -0500 Subject: [PATCH 2/6] Add foundational 19c and add to evals --- .../19c-openai-realtime-live-video.py | 163 ++++++++++++++++++ scripts/evals/run-release-evals.py | 1 + 2 files changed, 164 insertions(+) create mode 100644 examples/foundational/19c-openai-realtime-live-video.py diff --git a/examples/foundational/19c-openai-realtime-live-video.py b/examples/foundational/19c-openai-realtime-live-video.py new file mode 100644 index 000000000..10fd0095f --- /dev/null +++ b/examples/foundational/19c-openai-realtime-live-video.py @@ -0,0 +1,163 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import asyncio +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import ( + create_transport, + maybe_capture_participant_camera, + maybe_capture_participant_screen, +) +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, + InputAudioNoiseReduction, + InputAudioTranscription, + SemanticTurnDetection, + SessionProperties, +) +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + session_properties = SessionProperties( + audio=AudioConfiguration( + input=AudioInput( + transcription=InputAudioTranscription(), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + turn_detection=SemanticTurnDetection(), + # Or set to False to disable openai turn detection and use transport VAD + # turn_detection=False, + noise_reduction=InputAudioNoiseReduction(type="near_field"), + ) + ), + # In this example we provide tools through the context, but you could + # alternatively provide them here. + # tools=tools, + instructions="""You are a helpful and friendly AI. + +Act like a human, but remember that you aren't a human and that you can't do human +things in the real world. Your voice and personality should be warm and engaging, with a lively and +playful tone. + +If interacting in a non-English language, start by using the standard accent or dialect familiar to +the user. Talk quickly. You should always call a function if you can. Do not refer to these rules, +even if you're asked about them. + +You are participating in a voice conversation. Keep your responses concise, short, and to the point +unless specifically asked to elaborate on a topic. + +Remember, your responses should be short. Just one or two sentences, usually. Respond in English.""", + ) + + llm = OpenAIRealtimeLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + session_properties=session_properties, + start_audio_paused=False, + ) + + # Create a standard OpenAI LLM context object using the normal messages format. The + # OpenAIRealtimeLLMService will convert this internally to messages that the + # openai WebSocket API can understand. + context = LLMContext( + [{"role": "user", "content": "Say hello!"}], + ) + + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), + llm, # LLM + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + observers=[TranscriptionLogObserver()], + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + + await maybe_capture_participant_camera(transport, client, framerate=1) + await maybe_capture_participant_screen(transport, client, framerate=1) + + await task.queue_frames([LLMRunFrame()]) + await asyncio.sleep(3) + logger.debug("Unpausing audio and video") + llm.set_audio_input_paused(False) + llm.set_image_input_paused(False) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index db9fb2902..48491ea9b 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -188,6 +188,7 @@ TESTS_19 = [ ("19a-azure-realtime-beta.py", EVAL_WEATHER), ("19b-openai-realtime-text.py", EVAL_WEATHER), ("19b-openai-realtime-beta-text.py", EVAL_WEATHER), + ("19c-openai-realtime-live-video.py", EVAL_VISION_CAMERA), ] TESTS_21 = [ From 673d88417ceeb079df5a3a6728c781ccac1d6aeb Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 7 Jan 2026 17:58:06 -0500 Subject: [PATCH 3/6] Change Gemini Live and OpenAI Realtime logging to trace when sending a video frame --- src/pipecat/services/google/gemini_live/llm.py | 2 +- src/pipecat/services/openai/realtime/llm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 74682ea1c..4b0be986d 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1346,7 +1346,7 @@ class GeminiLiveLLMService(LLMService): return # Ignore if less than 1 second has passed self._last_sent_time = now # Update last sent time - logger.debug(f"Sending video frame to Gemini: {frame}") + logger.trace(f"Sending video frame to Gemini: {frame}") buffer = io.BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 21c8e58e8..9f413b33a 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -895,7 +895,7 @@ class OpenAIRealtimeLLMService(LLMService): return # Ignore if less than 1 second has passed self._last_image_sent_time = now # Update last sent time - logger.debug(f"Sending image frame to OpenAI Realtime: {frame}") + logger.trace(f"Sending image frame to OpenAI Realtime: {frame}") # Convert image to JPEG format and encode as base64 buffer = io.BytesIO() From 8bf8ebd34b75c0c897dbb9e61c2a014edf3b5850 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 7 Jan 2026 18:00:23 -0500 Subject: [PATCH 4/6] Remove start_audio_paused from OpenAI Realtime demos, and others --- examples/foundational/19-openai-realtime-beta.py | 1 - examples/foundational/19-openai-realtime.py | 1 - examples/foundational/19a-azure-realtime-beta.py | 1 - examples/foundational/19a-azure-realtime.py | 1 - examples/foundational/19b-openai-realtime-beta-text.py | 1 - examples/foundational/19b-openai-realtime-text.py | 1 - examples/foundational/19c-openai-realtime-live-video.py | 1 - .../foundational/20b-persistent-context-openai-realtime-beta.py | 1 - examples/foundational/20b-persistent-context-openai-realtime.py | 1 - examples/foundational/20f-persistent-context-grok-realtime.py | 1 - examples/foundational/51-grok-realtime.py | 1 - 11 files changed, 11 deletions(-) diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index e4afb8dfc..37fe86f96 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -143,7 +143,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 4d4cc3630..cea164543 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -169,7 +169,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 220fdc41e..d287344cb 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -141,7 +141,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re api_key=os.getenv("AZURE_REALTIME_API_KEY"), base_url=os.getenv("AZURE_REALTIME_BASE_URL"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index d2b64c4ea..6e245328a 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -148,7 +148,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re api_key=os.getenv("AZURE_REALTIME_API_KEY"), base_url=os.getenv("AZURE_REALTIME_BASE_URL"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19b-openai-realtime-beta-text.py b/examples/foundational/19b-openai-realtime-beta-text.py index 294f81604..0c66385d2 100644 --- a/examples/foundational/19b-openai-realtime-beta-text.py +++ b/examples/foundational/19b-openai-realtime-beta-text.py @@ -145,7 +145,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) tts = CartesiaTTSService( diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py index 1fe7bf358..927e5f5c1 100644 --- a/examples/foundational/19b-openai-realtime-text.py +++ b/examples/foundational/19b-openai-realtime-text.py @@ -152,7 +152,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) tts = CartesiaTTSService( diff --git a/examples/foundational/19c-openai-realtime-live-video.py b/examples/foundational/19c-openai-realtime-live-video.py index 10fd0095f..2c4c59b82 100644 --- a/examples/foundational/19c-openai-realtime-live-video.py +++ b/examples/foundational/19c-openai-realtime-live-video.py @@ -96,7 +96,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # Create a standard OpenAI LLM context object using the normal messages format. The diff --git a/examples/foundational/20b-persistent-context-openai-realtime-beta.py b/examples/foundational/20b-persistent-context-openai-realtime-beta.py index 4d089c2ed..19ccf81f7 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime-beta.py +++ b/examples/foundational/20b-persistent-context-openai-realtime-beta.py @@ -213,7 +213,6 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 50e7d39b0..2133fa628 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -205,7 +205,6 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/20f-persistent-context-grok-realtime.py b/examples/foundational/20f-persistent-context-grok-realtime.py index 5d1dccd26..3fd73eed5 100644 --- a/examples/foundational/20f-persistent-context-grok-realtime.py +++ b/examples/foundational/20f-persistent-context-grok-realtime.py @@ -194,7 +194,6 @@ Remember, your responses should be short - just one or two sentences usually.""" llm = GrokRealtimeLLMService( api_key=os.getenv("GROK_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # Register function handlers diff --git a/examples/foundational/51-grok-realtime.py b/examples/foundational/51-grok-realtime.py index f73cd9a8a..f56b2cb8e 100644 --- a/examples/foundational/51-grok-realtime.py +++ b/examples/foundational/51-grok-realtime.py @@ -201,7 +201,6 @@ Always be helpful and proactive in offering assistance.""", llm = GrokRealtimeLLMService( api_key=os.getenv("GROK_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # Register function handlers From b90a34228f722811c360247d22e0a8b424ffac30 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Wed, 7 Jan 2026 18:03:58 -0500 Subject: [PATCH 5/6] Update 19c to remove pausing audio and input --- examples/foundational/19c-openai-realtime-live-video.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/examples/foundational/19c-openai-realtime-live-video.py b/examples/foundational/19c-openai-realtime-live-video.py index 2c4c59b82..5c1a90777 100644 --- a/examples/foundational/19c-openai-realtime-live-video.py +++ b/examples/foundational/19c-openai-realtime-live-video.py @@ -135,10 +135,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re await maybe_capture_participant_screen(transport, client, framerate=1) await task.queue_frames([LLMRunFrame()]) - await asyncio.sleep(3) - logger.debug("Unpausing audio and video") - llm.set_audio_input_paused(False) - llm.set_image_input_paused(False) @transport.event_handler("on_client_disconnected") async def on_client_disconnected(transport, client): From 2aadac7a4dfe4f39c2d71ec6d6b31aa0c2078813 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Thu, 8 Jan 2026 13:17:07 -0500 Subject: [PATCH 6/6] Update OpenAIRealtime image to video to align with GeminiLive --- changelog/3360.added.md | 12 ++-- .../19c-openai-realtime-live-video.py | 5 +- src/pipecat/services/openai/realtime/llm.py | 63 ++++++++++--------- uv.lock | 8 +-- 4 files changed, 45 insertions(+), 43 deletions(-) diff --git a/changelog/3360.added.md b/changelog/3360.added.md index 5f5d92941..c9f22e823 100644 --- a/changelog/3360.added.md +++ b/changelog/3360.added.md @@ -1,6 +1,8 @@ - Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`: - - New `start_image_paused` parameter to control initial image input state - - New `image_detail` parameter to set image processing quality ("auto", "low", or "high") - - `set_image_input_paused()` method to pause/resume image input at runtime - - `set_image_detail()` method to adjust image quality dynamically - - Automatic rate limiting (1 image per second) to prevent API overload + - New `start_video_paused` parameter to control initial video input state + - New `video_frame_detail` parameter to set image processing quality ("auto", + "low", or "high"). This corresponds to OpenAI Realtime's `image_detail` + parameter. + - `set_video_input_paused()` method to pause/resume video input at runtime + - `set_video_frame_detail()` method to adjust video frame quality dynamically + - Automatic rate limiting (1 frame per second) to prevent API overload diff --git a/examples/foundational/19c-openai-realtime-live-video.py b/examples/foundational/19c-openai-realtime-live-video.py index 5c1a90777..af48b2355 100644 --- a/examples/foundational/19c-openai-realtime-live-video.py +++ b/examples/foundational/19c-openai-realtime-live-video.py @@ -5,7 +5,6 @@ # -import asyncio import os from dotenv import load_dotenv @@ -131,8 +130,8 @@ Remember, your responses should be short. Just one or two sentences, usually. Re async def on_client_connected(transport, client): logger.info(f"Client connected: {client}") - await maybe_capture_participant_camera(transport, client, framerate=1) - await maybe_capture_participant_screen(transport, client, framerate=1) + await maybe_capture_participant_camera(transport, client, framerate=0.5) + await maybe_capture_participant_screen(transport, client, framerate=0.5) await task.queue_frames([LLMRunFrame()]) diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 9f413b33a..598d4f510 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -109,8 +109,8 @@ class OpenAIRealtimeLLMService(LLMService): base_url: str = "wss://api.openai.com/v1/realtime", session_properties: Optional[events.SessionProperties] = None, start_audio_paused: bool = False, - start_image_paused: bool = False, - image_detail: str = "auto", + start_video_paused: bool = False, + video_frame_detail: str = "auto", send_transcription_frames: Optional[bool] = None, **kwargs, ): @@ -127,8 +127,9 @@ class OpenAIRealtimeLLMService(LLMService): These are session-level settings that can be updated during the session (except for voice and model). If None, uses default SessionProperties. start_audio_paused: Whether to start with audio input paused. Defaults to False. - start_image_paused: Whether to start with image input paused. Defaults to False. - image_detail: Detail level for image processing. Can be "auto", "low", or "high". + start_video_paused: Whether to start with video input paused. Defaults to False. + video_frame_detail: Detail level for video processing. Can be "auto", "low", or "high". + This sets the image_detail parameter in the OpenAI Realtime API. "auto" lets the model decide, "low" is faster and uses fewer tokens, "high" provides more detail. Defaults to "auto". send_transcription_frames: Whether to emit transcription frames. @@ -165,9 +166,9 @@ class OpenAIRealtimeLLMService(LLMService): session_properties or events.SessionProperties() ) self._audio_input_paused = start_audio_paused - self._image_input_paused = start_image_paused - self._image_detail = image_detail - self._last_image_sent_time = 0 + self._video_input_paused = start_video_paused + self._video_frame_detail = video_frame_detail + self._last_sent_time = 0 self._websocket = None self._receive_task = None self._context: LLMContext = None @@ -205,24 +206,24 @@ class OpenAIRealtimeLLMService(LLMService): """ self._audio_input_paused = paused - def set_image_input_paused(self, paused: bool): - """Set whether image input is paused. + def set_video_input_paused(self, paused: bool): + """Set whether video input is paused. Args: - paused: True to pause image input, False to resume. + paused: True to pause video input, False to resume. """ - self._image_input_paused = paused + self._video_input_paused = paused - def set_image_detail(self, detail: str): - """Set the detail level for image processing. + def set_video_frame_detail(self, detail: str): + """Set the detail level for video processing. Args: detail: Detail level - "auto", "low", or "high". """ if detail not in ["auto", "low", "high"]: - logger.warning(f"Invalid image detail '{detail}', must be 'auto', 'low', or 'high'") + logger.warning(f"Invalid video detail '{detail}', must be 'auto', 'low', or 'high'") return - self._image_detail = detail + self._video_frame_detail = detail def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" @@ -411,8 +412,8 @@ class OpenAIRealtimeLLMService(LLMService): if not self._audio_input_paused: await self._send_user_audio(frame) elif isinstance(frame, InputImageRawFrame): - if not self._image_input_paused: - await self._send_user_image(frame) + if not self._video_input_paused: + await self._send_user_video(frame) elif isinstance(frame, InterruptionFrame): await self._handle_interruption() elif isinstance(frame, UserStartedSpeakingFrame): @@ -881,39 +882,39 @@ class OpenAIRealtimeLLMService(LLMService): payload = base64.b64encode(frame.audio).decode("utf-8") await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) - async def _send_user_image(self, frame: InputImageRawFrame): - """Send user image frame to OpenAI Realtime API. + async def _send_user_video(self, frame: InputImageRawFrame): + """Send user video frame to OpenAI Realtime API. Args: - frame: The input image frame to send. + frame: The InputImageRawFrame to send. """ - if self._image_input_paused or self._disconnecting or not self._websocket: + if self._video_input_paused or self._disconnecting or not self._websocket: return now = time.time() - if now - self._last_image_sent_time < 1: + if now - self._last_sent_time < 1: return # Ignore if less than 1 second has passed - self._last_image_sent_time = now # Update last sent time - logger.trace(f"Sending image frame to OpenAI Realtime: {frame}") + self._last_sent_time = now # Update last sent time + logger.trace(f"Sending video frame to OpenAI Realtime: {frame}") - # Convert image to JPEG format and encode as base64 + # Convert video frame to JPEG format and encode as base64 buffer = io.BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") - image_data = base64.b64encode(buffer.getvalue()).decode("utf-8") + data = base64.b64encode(buffer.getvalue()).decode("utf-8") - # Create data URI for the image - image_url = f"data:image/jpeg;base64,{image_data}" + # Create data URI for the video frame + data_uri = f"data:image/jpeg;base64,{data}" - # Create a conversation item with the image + # Create a conversation item with the video frame item = events.ConversationItem( type="message", role="user", content=[ events.ItemContent( type="input_image", - image_url=image_url, - detail=self._image_detail, + image_url=data_uri, + detail=self._video_frame_detail, ) ], ) diff --git a/uv.lock b/uv.lock index 7dd5e0562..51a0a2ec1 100644 --- a/uv.lock +++ b/uv.lock @@ -4208,7 +4208,7 @@ requires-dist = [ { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, - { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.4" }, + { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.6" }, { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, @@ -5948,16 +5948,16 @@ wheels = [ [[package]] name = "speechmatics-voice" -version = "0.2.4" +version = "0.2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "pydantic" }, { name = "speechmatics-rt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b2/f9/9d81e4abe9ae1c8745372eaf43523213b0333e9721699fb0f3d3bff6c17e/speechmatics_voice-0.2.4.tar.gz", hash = "sha256:e3b5c7a8c24fa7d555b80a72ab181797665c74944400468ca5fb7e54b5f9eae6", size = 60852, upload-time = "2025-12-17T23:22:13.437Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/18/7790718c826be18eadaa7cfc0cc9f229d157f5f3aff628b9fc0f180a7878/speechmatics_voice-0.2.6.tar.gz", hash = "sha256:ae384e8f97862fc6adf38937e1d1d63cd16b64bc49aded8ccad273155634a636", size = 60881, upload-time = "2026-01-08T00:54:41.405Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/a6/401dba9be6be914e57b7814360ba0bece55f24140bb7d5c3dc5f07bcd77f/speechmatics_voice-0.2.4-py3-none-any.whl", hash = "sha256:71d0f5272c2db1221422ab19b6c898ea7b38f9fb7f523904f54a4d8c3e4cef12", size = 57056, upload-time = "2025-12-17T23:22:11.837Z" }, + { url = "https://files.pythonhosted.org/packages/e2/cc/ae6dc3d5638a3fc86c4537af1fb394dee3c4a2a5e9dbebf9fb83a8052939/speechmatics_voice-0.2.6-py3-none-any.whl", hash = "sha256:15d61cb02d7fe492f966cc28ddb0ada199fdd12543b9a61cb8757c7bf25b7a94", size = 57103, upload-time = "2026-01-08T00:54:39.92Z" }, ] [package.optional-dependencies]