"""Shared camera-frame analysis for Pipeline and Realtime runtimes.""" from __future__ import annotations import asyncio import base64 from io import BytesIO from models import AssistantConfig, RuntimeModelResource from openai import AsyncOpenAI from PIL import Image from pipecat.frames.frames import UserImageRawFrame VISION_TOOL_NAME = "fetch_user_image" VISION_SYSTEM_HINT = ( "当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/" "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧," "再基于画面回答。" ) VISION_ANALYSIS_SYSTEM_PROMPT = ( "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。" "如果画面不足以判断,请明确说明不确定。" ) def _require(value: str, label: str) -> str: if value: return value raise ValueError(f"缺少模型资源配置: {label}") def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes: """Encode one Pipecat camera frame as a storage- and model-ready JPEG.""" if not frame.format: raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型") buffer = BytesIO() Image.frombytes(frame.format, frame.size, frame.image).save( buffer, format="JPEG", quality=85, ) return buffer.getvalue() def image_data_uri(frame: UserImageRawFrame) -> str: """Encode one Pipecat camera frame for a vision chat-completion request.""" encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8") return f"data:image/jpeg;base64,{encoded}" async def analyze_image_with_vision_model( cfg: AssistantConfig, frame: UserImageRawFrame, question: str, ) -> str: """Analyze one frame with the configured independent vision LLM.""" if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}: raise ValueError( f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}" ) data_uri = await asyncio.to_thread(image_data_uri, frame) extra_body = cfg.vision_llm_values.get("extraBody") extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} client = AsyncOpenAI( api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"), base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"), ) try: response = await client.chat.completions.create( model=_require(cfg.vision_model, "Vision LLM modelId"), messages=[ {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT}, { "role": "user", "content": [ {"type": "text", "text": question}, {"type": "image_url", "image_url": {"url": data_uri}}, ], }, ], **extra, ) finally: await client.close() content = response.choices[0].message.content if response.choices else "" if isinstance(content, str): return content.strip() return str(content or "").strip() def config_with_vision_resource( cfg: AssistantConfig, resource: RuntimeModelResource, ) -> AssistantConfig: """Create the config view used to analyze one Workflow camera frame.""" if resource.capability != "LLM": raise ValueError(f"视觉模型资源能力无效:{resource.capability}") if not resource.support_image_input: raise ValueError(f"视觉模型不支持图片输入:{resource.id}") result = cfg.model_copy(deep=True) values = resource.values or {} secrets = resource.secrets or {} result.vision_model_resource_id = resource.id result.vision_model = str(values.get("modelId") or "") result.vision_llm_interface_type = resource.interface_type result.vision_llm_values = values result.vision_llm_secrets = secrets result.vision_llm_support_image_input = resource.support_image_input result.vision_llm_api_key = str(secrets.get("apiKey") or "") result.vision_llm_base_url = str(values.get("apiUrl") or "") return result def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig: """Legacy fallback when a Workflow LLM is not present in the resource map.""" if not cfg.llm_support_image_input: raise ValueError("当前大语言模型不支持图片输入") result = cfg.model_copy(deep=True) result.vision_model_resource_id = None result.vision_model = cfg.model result.vision_llm_interface_type = cfg.llm_interface_type result.vision_llm_values = dict(cfg.llm_values) result.vision_llm_secrets = dict(cfg.llm_secrets) result.vision_llm_support_image_input = cfg.llm_support_image_input result.vision_llm_api_key = cfg.llm_api_key result.vision_llm_base_url = cfg.llm_base_url return result