Add vision model support and related configurations in Assistant

- Introduce new fields in AssistantConfig, schemas, and database models to support vision capabilities, including `vision_enabled` and `vision_model_resource_id`.
- Enhance validation logic in routes to ensure proper handling of vision models and their requirements.
- Update the AssistantPage and related frontend components to include options for enabling vision understanding and selecting appropriate vision models.
- Modify database seed scripts to include vision-related data for assistants, ensuring consistent setup.
- Refactor related functions to integrate vision model handling in the audio-visual processing pipeline.
This commit is contained in:
Xin Wang
2026-07-07 21:50:15 +08:00
parent c51a70e134
commit 5f71bf1681
10 changed files with 180 additions and 41 deletions

View File

@@ -58,6 +58,11 @@ async def resolve_runtime_config(
stt_resource = await _resource_for(session, assistant.id, "ASR")
tts_resource = await _resource_for(session, assistant.id, "TTS")
realtime_resource = await _resource_for(session, assistant.id, "Realtime")
vision_resource = (
await session.get(ModelResource, assistant.vision_model_resource_id)
if assistant.vision_model_resource_id
else llm_resource
)
return AssistantConfig(
name=assistant.name,
@@ -89,6 +94,17 @@ async def resolve_runtime_config(
llm_support_image_input=(
bool(llm_resource.support_image_input) if llm_resource else False
),
vision_enabled=assistant.vision_enabled,
vision_model_resource_id=assistant.vision_model_resource_id,
vision_model=str(_value(vision_resource, "modelId", "")),
vision_llm_interface_type=(
vision_resource.interface_type if vision_resource else "openai-llm"
),
vision_llm_values=(vision_resource.values or {}) if vision_resource else {},
vision_llm_secrets=(vision_resource.secrets or {}) if vision_resource else {},
vision_llm_support_image_input=(
bool(vision_resource.support_image_input) if vision_resource else False
),
stt_values=(stt_resource.values or {}) if stt_resource else {},
stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
tts_values=(tts_resource.values or {}) if tts_resource else {},
@@ -101,6 +117,12 @@ async def resolve_runtime_config(
# 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底
llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY),
llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)),
vision_llm_api_key=_secret(
vision_resource, "apiKey", config.LLM_API_KEY
),
vision_llm_base_url=str(
_value(vision_resource, "apiUrl", config.LLM_BASE_URL)
),
stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY),
stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)),
tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY),