diff --git a/changelog/3360.added.md b/changelog/3360.added.md new file mode 100644 index 000000000..c9f22e823 --- /dev/null +++ b/changelog/3360.added.md @@ -0,0 +1,8 @@ +- Added image support to `OpenAIRealtimeLLMService` via `InputImageRawFrame`: + - New `start_video_paused` parameter to control initial video input state + - New `video_frame_detail` parameter to set image processing quality ("auto", + "low", or "high"). This corresponds to OpenAI Realtime's `image_detail` + parameter. + - `set_video_input_paused()` method to pause/resume video input at runtime + - `set_video_frame_detail()` method to adjust video frame quality dynamically + - Automatic rate limiting (1 frame per second) to prevent API overload diff --git a/examples/foundational/19-openai-realtime-beta.py b/examples/foundational/19-openai-realtime-beta.py index e4afb8dfc..37fe86f96 100644 --- a/examples/foundational/19-openai-realtime-beta.py +++ b/examples/foundational/19-openai-realtime-beta.py @@ -143,7 +143,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19-openai-realtime.py b/examples/foundational/19-openai-realtime.py index 4d4cc3630..cea164543 100644 --- a/examples/foundational/19-openai-realtime.py +++ b/examples/foundational/19-openai-realtime.py @@ -169,7 +169,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19a-azure-realtime-beta.py b/examples/foundational/19a-azure-realtime-beta.py index 220fdc41e..d287344cb 100644 --- a/examples/foundational/19a-azure-realtime-beta.py +++ b/examples/foundational/19a-azure-realtime-beta.py @@ -141,7 +141,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re api_key=os.getenv("AZURE_REALTIME_API_KEY"), base_url=os.getenv("AZURE_REALTIME_BASE_URL"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19a-azure-realtime.py b/examples/foundational/19a-azure-realtime.py index d2b64c4ea..6e245328a 100644 --- a/examples/foundational/19a-azure-realtime.py +++ b/examples/foundational/19a-azure-realtime.py @@ -148,7 +148,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re api_key=os.getenv("AZURE_REALTIME_API_KEY"), base_url=os.getenv("AZURE_REALTIME_BASE_URL"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/19b-openai-realtime-beta-text.py b/examples/foundational/19b-openai-realtime-beta-text.py index 294f81604..0c66385d2 100644 --- a/examples/foundational/19b-openai-realtime-beta-text.py +++ b/examples/foundational/19b-openai-realtime-beta-text.py @@ -145,7 +145,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) tts = CartesiaTTSService( diff --git a/examples/foundational/19b-openai-realtime-text.py b/examples/foundational/19b-openai-realtime-text.py index 1fe7bf358..927e5f5c1 100644 --- a/examples/foundational/19b-openai-realtime-text.py +++ b/examples/foundational/19b-openai-realtime-text.py @@ -152,7 +152,6 @@ Remember, your responses should be short. Just one or two sentences, usually. Re llm = OpenAIRealtimeLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) tts = CartesiaTTSService( diff --git a/examples/foundational/19c-openai-realtime-live-video.py b/examples/foundational/19c-openai-realtime-live-video.py new file mode 100644 index 000000000..af48b2355 --- /dev/null +++ b/examples/foundational/19c-openai-realtime-live-video.py @@ -0,0 +1,157 @@ +# +# Copyright (c) 2024–2025, Daily +# +# SPDX-License-Identifier: BSD 2-Clause License +# + + +import os + +from dotenv import load_dotenv +from loguru import logger + +from pipecat.audio.vad.silero import SileroVADAnalyzer +from pipecat.frames.frames import LLMRunFrame +from pipecat.observers.loggers.transcription_log_observer import TranscriptionLogObserver +from pipecat.pipeline.pipeline import Pipeline +from pipecat.pipeline.runner import PipelineRunner +from pipecat.pipeline.task import PipelineParams, PipelineTask +from pipecat.processors.aggregators.llm_context import LLMContext +from pipecat.processors.aggregators.llm_response_universal import LLMContextAggregatorPair +from pipecat.runner.types import RunnerArguments +from pipecat.runner.utils import ( + create_transport, + maybe_capture_participant_camera, + maybe_capture_participant_screen, +) +from pipecat.services.openai.realtime.events import ( + AudioConfiguration, + AudioInput, + InputAudioNoiseReduction, + InputAudioTranscription, + SemanticTurnDetection, + SessionProperties, +) +from pipecat.services.openai.realtime.llm import OpenAIRealtimeLLMService +from pipecat.transports.base_transport import BaseTransport, TransportParams +from pipecat.transports.daily.transport import DailyParams + +load_dotenv(override=True) + + +# We store functions so objects (e.g. SileroVADAnalyzer) don't get +# instantiated. The function will be called when the desired transport gets +# selected. +transport_params = { + "daily": lambda: DailyParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), + "webrtc": lambda: TransportParams( + audio_in_enabled=True, + audio_out_enabled=True, + video_in_enabled=True, + vad_analyzer=SileroVADAnalyzer(), + ), +} + + +async def run_bot(transport: BaseTransport, runner_args: RunnerArguments): + logger.info(f"Starting bot") + + session_properties = SessionProperties( + audio=AudioConfiguration( + input=AudioInput( + transcription=InputAudioTranscription(), + # Set openai TurnDetection parameters. Not setting this at all will turn it + # on by default + turn_detection=SemanticTurnDetection(), + # Or set to False to disable openai turn detection and use transport VAD + # turn_detection=False, + noise_reduction=InputAudioNoiseReduction(type="near_field"), + ) + ), + # In this example we provide tools through the context, but you could + # alternatively provide them here. + # tools=tools, + instructions="""You are a helpful and friendly AI. + +Act like a human, but remember that you aren't a human and that you can't do human +things in the real world. Your voice and personality should be warm and engaging, with a lively and +playful tone. + +If interacting in a non-English language, start by using the standard accent or dialect familiar to +the user. Talk quickly. You should always call a function if you can. Do not refer to these rules, +even if you're asked about them. + +You are participating in a voice conversation. Keep your responses concise, short, and to the point +unless specifically asked to elaborate on a topic. + +Remember, your responses should be short. Just one or two sentences, usually. Respond in English.""", + ) + + llm = OpenAIRealtimeLLMService( + api_key=os.getenv("OPENAI_API_KEY"), + session_properties=session_properties, + ) + + # Create a standard OpenAI LLM context object using the normal messages format. The + # OpenAIRealtimeLLMService will convert this internally to messages that the + # openai WebSocket API can understand. + context = LLMContext( + [{"role": "user", "content": "Say hello!"}], + ) + + context_aggregator = LLMContextAggregatorPair(context) + + pipeline = Pipeline( + [ + transport.input(), # Transport user input + context_aggregator.user(), + llm, # LLM + transport.output(), # Transport bot output + context_aggregator.assistant(), + ] + ) + + task = PipelineTask( + pipeline, + params=PipelineParams( + enable_metrics=True, + enable_usage_metrics=True, + ), + idle_timeout_secs=runner_args.pipeline_idle_timeout_secs, + observers=[TranscriptionLogObserver()], + ) + + @transport.event_handler("on_client_connected") + async def on_client_connected(transport, client): + logger.info(f"Client connected: {client}") + + await maybe_capture_participant_camera(transport, client, framerate=0.5) + await maybe_capture_participant_screen(transport, client, framerate=0.5) + + await task.queue_frames([LLMRunFrame()]) + + @transport.event_handler("on_client_disconnected") + async def on_client_disconnected(transport, client): + logger.info(f"Client disconnected") + await task.cancel() + + runner = PipelineRunner(handle_sigint=runner_args.handle_sigint) + + await runner.run(task) + + +async def bot(runner_args: RunnerArguments): + """Main bot entry point compatible with Pipecat Cloud.""" + transport = await create_transport(runner_args, transport_params) + await run_bot(transport, runner_args) + + +if __name__ == "__main__": + from pipecat.runner.run import main + + main() diff --git a/examples/foundational/20b-persistent-context-openai-realtime-beta.py b/examples/foundational/20b-persistent-context-openai-realtime-beta.py index 4d089c2ed..19ccf81f7 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime-beta.py +++ b/examples/foundational/20b-persistent-context-openai-realtime-beta.py @@ -213,7 +213,6 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeBetaLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/20b-persistent-context-openai-realtime.py b/examples/foundational/20b-persistent-context-openai-realtime.py index 50e7d39b0..2133fa628 100644 --- a/examples/foundational/20b-persistent-context-openai-realtime.py +++ b/examples/foundational/20b-persistent-context-openai-realtime.py @@ -205,7 +205,6 @@ Remember, your responses should be short. Just one or two sentences, usually.""" llm = OpenAIRealtimeLLMService( api_key=os.getenv("OPENAI_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # you can either register a single function for all function calls, or specific functions diff --git a/examples/foundational/20f-persistent-context-grok-realtime.py b/examples/foundational/20f-persistent-context-grok-realtime.py index 5d1dccd26..3fd73eed5 100644 --- a/examples/foundational/20f-persistent-context-grok-realtime.py +++ b/examples/foundational/20f-persistent-context-grok-realtime.py @@ -194,7 +194,6 @@ Remember, your responses should be short - just one or two sentences usually.""" llm = GrokRealtimeLLMService( api_key=os.getenv("GROK_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # Register function handlers diff --git a/examples/foundational/51-grok-realtime.py b/examples/foundational/51-grok-realtime.py index f73cd9a8a..f56b2cb8e 100644 --- a/examples/foundational/51-grok-realtime.py +++ b/examples/foundational/51-grok-realtime.py @@ -201,7 +201,6 @@ Always be helpful and proactive in offering assistance.""", llm = GrokRealtimeLLMService( api_key=os.getenv("GROK_API_KEY"), session_properties=session_properties, - start_audio_paused=False, ) # Register function handlers diff --git a/scripts/evals/run-release-evals.py b/scripts/evals/run-release-evals.py index db9fb2902..48491ea9b 100644 --- a/scripts/evals/run-release-evals.py +++ b/scripts/evals/run-release-evals.py @@ -188,6 +188,7 @@ TESTS_19 = [ ("19a-azure-realtime-beta.py", EVAL_WEATHER), ("19b-openai-realtime-text.py", EVAL_WEATHER), ("19b-openai-realtime-beta-text.py", EVAL_WEATHER), + ("19c-openai-realtime-live-video.py", EVAL_VISION_CAMERA), ] TESTS_21 = [ diff --git a/src/pipecat/services/google/gemini_live/llm.py b/src/pipecat/services/google/gemini_live/llm.py index 74682ea1c..4b0be986d 100644 --- a/src/pipecat/services/google/gemini_live/llm.py +++ b/src/pipecat/services/google/gemini_live/llm.py @@ -1346,7 +1346,7 @@ class GeminiLiveLLMService(LLMService): return # Ignore if less than 1 second has passed self._last_sent_time = now # Update last sent time - logger.debug(f"Sending video frame to Gemini: {frame}") + logger.trace(f"Sending video frame to Gemini: {frame}") buffer = io.BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") diff --git a/src/pipecat/services/openai/realtime/events.py b/src/pipecat/services/openai/realtime/events.py index 09d1e8ae4..c0ef5b890 100644 --- a/src/pipecat/services/openai/realtime/events.py +++ b/src/pipecat/services/openai/realtime/events.py @@ -217,16 +217,22 @@ class ItemContent(BaseModel): """Content within a conversation item. Parameters: - type: Content type (text, audio, input_text, input_audio, output_text, or output_audio). + type: Content type (text, audio, input_text, input_audio, input_image, output_text, or output_audio). text: Text content for text-based items. audio: Base64-encoded audio data for audio items. transcript: Transcribed text for audio items. + image_url: Base64-encoded image data as a data URI for input_image items. + detail: Detail level for image processing ("auto", "low", or "high"). """ - type: Literal["text", "audio", "input_text", "input_audio", "output_text", "output_audio"] + type: Literal[ + "text", "audio", "input_text", "input_audio", "input_image", "output_text", "output_audio" + ] text: Optional[str] = None audio: Optional[str] = None # base64-encoded audio transcript: Optional[str] = None + image_url: Optional[str] = None # base64-encoded image as data URI + detail: Optional[Literal["auto", "low", "high"]] = None class ConversationItem(BaseModel): diff --git a/src/pipecat/services/openai/realtime/llm.py b/src/pipecat/services/openai/realtime/llm.py index 5cb50e16d..598d4f510 100644 --- a/src/pipecat/services/openai/realtime/llm.py +++ b/src/pipecat/services/openai/realtime/llm.py @@ -7,12 +7,14 @@ """OpenAI Realtime LLM service implementation with WebSocket support.""" import base64 +import io import json import time from dataclasses import dataclass from typing import Optional from loguru import logger +from PIL import Image from pipecat.adapters.schemas.tools_schema import ToolsSchema from pipecat.adapters.services.open_ai_realtime_adapter import ( @@ -25,6 +27,7 @@ from pipecat.frames.frames import ( EndFrame, Frame, InputAudioRawFrame, + InputImageRawFrame, InterimTranscriptionFrame, InterruptionFrame, LLMContextFrame, @@ -106,6 +109,8 @@ class OpenAIRealtimeLLMService(LLMService): base_url: str = "wss://api.openai.com/v1/realtime", session_properties: Optional[events.SessionProperties] = None, start_audio_paused: bool = False, + start_video_paused: bool = False, + video_frame_detail: str = "auto", send_transcription_frames: Optional[bool] = None, **kwargs, ): @@ -122,6 +127,11 @@ class OpenAIRealtimeLLMService(LLMService): These are session-level settings that can be updated during the session (except for voice and model). If None, uses default SessionProperties. start_audio_paused: Whether to start with audio input paused. Defaults to False. + start_video_paused: Whether to start with video input paused. Defaults to False. + video_frame_detail: Detail level for video processing. Can be "auto", "low", or "high". + This sets the image_detail parameter in the OpenAI Realtime API. + "auto" lets the model decide, "low" is faster and uses fewer tokens, + "high" provides more detail. Defaults to "auto". send_transcription_frames: Whether to emit transcription frames. .. deprecated:: 0.0.92 @@ -156,6 +166,9 @@ class OpenAIRealtimeLLMService(LLMService): session_properties or events.SessionProperties() ) self._audio_input_paused = start_audio_paused + self._video_input_paused = start_video_paused + self._video_frame_detail = video_frame_detail + self._last_sent_time = 0 self._websocket = None self._receive_task = None self._context: LLMContext = None @@ -193,6 +206,25 @@ class OpenAIRealtimeLLMService(LLMService): """ self._audio_input_paused = paused + def set_video_input_paused(self, paused: bool): + """Set whether video input is paused. + + Args: + paused: True to pause video input, False to resume. + """ + self._video_input_paused = paused + + def set_video_frame_detail(self, detail: str): + """Set the detail level for video processing. + + Args: + detail: Detail level - "auto", "low", or "high". + """ + if detail not in ["auto", "low", "high"]: + logger.warning(f"Invalid video detail '{detail}', must be 'auto', 'low', or 'high'") + return + self._video_frame_detail = detail + def _is_modality_enabled(self, modality: str) -> bool: """Check if a specific modality is enabled, "text" or "audio".""" modalities = self._session_properties.output_modalities or ["audio", "text"] @@ -379,6 +411,9 @@ class OpenAIRealtimeLLMService(LLMService): elif isinstance(frame, InputAudioRawFrame): if not self._audio_input_paused: await self._send_user_audio(frame) + elif isinstance(frame, InputImageRawFrame): + if not self._video_input_paused: + await self._send_user_video(frame) elif isinstance(frame, InterruptionFrame): await self._handle_interruption() elif isinstance(frame, UserStartedSpeakingFrame): @@ -847,6 +882,49 @@ class OpenAIRealtimeLLMService(LLMService): payload = base64.b64encode(frame.audio).decode("utf-8") await self.send_client_event(events.InputAudioBufferAppendEvent(audio=payload)) + async def _send_user_video(self, frame: InputImageRawFrame): + """Send user video frame to OpenAI Realtime API. + + Args: + frame: The InputImageRawFrame to send. + """ + if self._video_input_paused or self._disconnecting or not self._websocket: + return + + now = time.time() + if now - self._last_sent_time < 1: + return # Ignore if less than 1 second has passed + + self._last_sent_time = now # Update last sent time + logger.trace(f"Sending video frame to OpenAI Realtime: {frame}") + + # Convert video frame to JPEG format and encode as base64 + buffer = io.BytesIO() + Image.frombytes(frame.format, frame.size, frame.image).save(buffer, format="JPEG") + data = base64.b64encode(buffer.getvalue()).decode("utf-8") + + # Create data URI for the video frame + data_uri = f"data:image/jpeg;base64,{data}" + + # Create a conversation item with the video frame + item = events.ConversationItem( + type="message", + role="user", + content=[ + events.ItemContent( + type="input_image", + image_url=data_uri, + detail=self._video_frame_detail, + ) + ], + ) + + # Send the conversation item + try: + await self.send_client_event(events.ConversationItemCreateEvent(item=item)) + except Exception as e: + await self.push_error(error_msg=f"Send error: {e}") + async def _send_tool_result(self, tool_call_id: str, result: str): item = events.ConversationItem( type="function_call_output", diff --git a/uv.lock b/uv.lock index 7dd5e0562..51a0a2ec1 100644 --- a/uv.lock +++ b/uv.lock @@ -4208,7 +4208,7 @@ requires-dist = [ { name = "simli-ai", marker = "extra == 'simli'", specifier = "~=1.0.3" }, { name = "soundfile", marker = "extra == 'soundfile'", specifier = "~=0.13.1" }, { name = "soxr", specifier = "~=0.5.0" }, - { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.4" }, + { name = "speechmatics-voice", extras = ["smart"], marker = "extra == 'speechmatics'", specifier = ">=0.2.6" }, { name = "strands-agents", marker = "extra == 'strands'", specifier = ">=1.9.1,<2" }, { name = "tenacity", marker = "extra == 'livekit'", specifier = ">=8.2.3,<10.0.0" }, { name = "timm", marker = "extra == 'moondream'", specifier = "~=1.0.13" }, @@ -5948,16 +5948,16 @@ wheels = [ [[package]] name = "speechmatics-voice" -version = "0.2.4" +version = "0.2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy" }, { name = "pydantic" }, { name = "speechmatics-rt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b2/f9/9d81e4abe9ae1c8745372eaf43523213b0333e9721699fb0f3d3bff6c17e/speechmatics_voice-0.2.4.tar.gz", hash = "sha256:e3b5c7a8c24fa7d555b80a72ab181797665c74944400468ca5fb7e54b5f9eae6", size = 60852, upload-time = "2025-12-17T23:22:13.437Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d2/18/7790718c826be18eadaa7cfc0cc9f229d157f5f3aff628b9fc0f180a7878/speechmatics_voice-0.2.6.tar.gz", hash = "sha256:ae384e8f97862fc6adf38937e1d1d63cd16b64bc49aded8ccad273155634a636", size = 60881, upload-time = "2026-01-08T00:54:41.405Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/a6/401dba9be6be914e57b7814360ba0bece55f24140bb7d5c3dc5f07bcd77f/speechmatics_voice-0.2.4-py3-none-any.whl", hash = "sha256:71d0f5272c2db1221422ab19b6c898ea7b38f9fb7f523904f54a4d8c3e4cef12", size = 57056, upload-time = "2025-12-17T23:22:11.837Z" }, + { url = "https://files.pythonhosted.org/packages/e2/cc/ae6dc3d5638a3fc86c4537af1fb394dee3c4a2a5e9dbebf9fb83a8052939/speechmatics_voice-0.2.6-py3-none-any.whl", hash = "sha256:15d61cb02d7fe492f966cc28ddb0ada199fdd12543b9a61cb8757c7bf25b7a94", size = 57103, upload-time = "2026-01-08T00:54:39.92Z" }, ] [package.optional-dependencies]