"""Shared vision capability metadata and Workflow model resolution.""" from __future__ import annotations from models import AssistantConfig, RuntimeModelResource VISION_TOOL_NAME = "fetch_user_image" VISION_SYSTEM_HINT = ( "当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/" "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧," "再基于画面回答。" ) VISION_ANALYSIS_SYSTEM_PROMPT = ( "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。" "如果画面不足以判断,请明确说明不确定。" ) def config_with_vision_resource( cfg: AssistantConfig, resource: RuntimeModelResource, ) -> AssistantConfig: """Create the config view used to analyze one Workflow camera frame.""" if resource.capability != "LLM": raise ValueError(f"视觉模型资源能力无效:{resource.capability}") if not resource.support_image_input: raise ValueError(f"视觉模型不支持图片输入:{resource.id}") result = cfg.model_copy(deep=True) values = resource.values or {} secrets = resource.secrets or {} result.vision_model_resource_id = resource.id result.vision_model = str(values.get("modelId") or "") result.vision_llm_interface_type = resource.interface_type result.vision_llm_values = values result.vision_llm_secrets = secrets result.vision_llm_support_image_input = resource.support_image_input result.vision_llm_api_key = str(secrets.get("apiKey") or "") result.vision_llm_base_url = str(values.get("apiUrl") or "") return result def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig: """Legacy fallback when a Workflow LLM is not present in the resource map.""" if not cfg.llm_support_image_input: raise ValueError("当前大语言模型不支持图片输入") result = cfg.model_copy(deep=True) result.vision_model_resource_id = None result.vision_model = cfg.model result.vision_llm_interface_type = cfg.llm_interface_type result.vision_llm_values = dict(cfg.llm_values) result.vision_llm_secrets = dict(cfg.llm_secrets) result.vision_llm_support_image_input = cfg.llm_support_image_input result.vision_llm_api_key = cfg.llm_api_key result.vision_llm_base_url = cfg.llm_base_url return result