59 lines
2.4 KiB
Python
59 lines
2.4 KiB
Python
"""Shared vision capability metadata and Workflow model resolution."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from models import AssistantConfig, RuntimeModelResource
|
|
|
|
|
|
VISION_TOOL_NAME = "fetch_user_image"
|
|
VISION_SYSTEM_HINT = (
|
|
"当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
|
|
"环境状态或需要你看一眼时，调用 fetch_user_image 获取当前视频帧，"
|
|
"再基于画面回答。"
|
|
)
|
|
VISION_ANALYSIS_SYSTEM_PROMPT = (
|
|
"你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
|
|
"如果画面不足以判断，请明确说明不确定。"
|
|
)
|
|
|
|
|
|
def config_with_vision_resource(
|
|
cfg: AssistantConfig,
|
|
resource: RuntimeModelResource,
|
|
) -> AssistantConfig:
|
|
"""Create the config view used to analyze one Workflow camera frame."""
|
|
if resource.capability != "LLM":
|
|
raise ValueError(f"视觉模型资源能力无效:{resource.capability}")
|
|
if not resource.support_image_input:
|
|
raise ValueError(f"视觉模型不支持图片输入:{resource.id}")
|
|
|
|
result = cfg.model_copy(deep=True)
|
|
values = resource.values or {}
|
|
secrets = resource.secrets or {}
|
|
result.vision_model_resource_id = resource.id
|
|
result.vision_model = str(values.get("modelId") or "")
|
|
result.vision_llm_interface_type = resource.interface_type
|
|
result.vision_llm_values = values
|
|
result.vision_llm_secrets = secrets
|
|
result.vision_llm_support_image_input = resource.support_image_input
|
|
result.vision_llm_api_key = str(secrets.get("apiKey") or "")
|
|
result.vision_llm_base_url = str(values.get("apiUrl") or "")
|
|
return result
|
|
|
|
|
|
def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig:
|
|
"""Legacy fallback when a Workflow LLM is not present in the resource map."""
|
|
if not cfg.llm_support_image_input:
|
|
raise ValueError("当前大语言模型不支持图片输入")
|
|
|
|
result = cfg.model_copy(deep=True)
|
|
result.vision_model_resource_id = None
|
|
result.vision_model = cfg.model
|
|
result.vision_llm_interface_type = cfg.llm_interface_type
|
|
result.vision_llm_values = dict(cfg.llm_values)
|
|
result.vision_llm_secrets = dict(cfg.llm_secrets)
|
|
result.vision_llm_support_image_input = cfg.llm_support_image_input
|
|
result.vision_llm_api_key = cfg.llm_api_key
|
|
result.vision_llm_base_url = cfg.llm_base_url
|
|
return result
|