Add vision model support and related configurations in Assistant
- Introduce new fields in AssistantConfig, schemas, and database models to support vision capabilities, including `vision_enabled` and `vision_model_resource_id`. - Enhance validation logic in routes to ensure proper handling of vision models and their requirements. - Update the AssistantPage and related frontend components to include options for enabling vision understanding and selecting appropriate vision models. - Modify database seed scripts to include vision-related data for assistants, ensuring consistent setup. - Refactor related functions to integrate vision model handling in the audio-visual processing pipeline.
This commit is contained in:
@@ -58,6 +58,11 @@ async def resolve_runtime_config(
|
||||
stt_resource = await _resource_for(session, assistant.id, "ASR")
|
||||
tts_resource = await _resource_for(session, assistant.id, "TTS")
|
||||
realtime_resource = await _resource_for(session, assistant.id, "Realtime")
|
||||
vision_resource = (
|
||||
await session.get(ModelResource, assistant.vision_model_resource_id)
|
||||
if assistant.vision_model_resource_id
|
||||
else llm_resource
|
||||
)
|
||||
|
||||
return AssistantConfig(
|
||||
name=assistant.name,
|
||||
@@ -89,6 +94,17 @@ async def resolve_runtime_config(
|
||||
llm_support_image_input=(
|
||||
bool(llm_resource.support_image_input) if llm_resource else False
|
||||
),
|
||||
vision_enabled=assistant.vision_enabled,
|
||||
vision_model_resource_id=assistant.vision_model_resource_id,
|
||||
vision_model=str(_value(vision_resource, "modelId", "")),
|
||||
vision_llm_interface_type=(
|
||||
vision_resource.interface_type if vision_resource else "openai-llm"
|
||||
),
|
||||
vision_llm_values=(vision_resource.values or {}) if vision_resource else {},
|
||||
vision_llm_secrets=(vision_resource.secrets or {}) if vision_resource else {},
|
||||
vision_llm_support_image_input=(
|
||||
bool(vision_resource.support_image_input) if vision_resource else False
|
||||
),
|
||||
stt_values=(stt_resource.values or {}) if stt_resource else {},
|
||||
stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
|
||||
tts_values=(tts_resource.values or {}) if tts_resource else {},
|
||||
@@ -101,6 +117,12 @@ async def resolve_runtime_config(
|
||||
# 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底
|
||||
llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY),
|
||||
llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)),
|
||||
vision_llm_api_key=_secret(
|
||||
vision_resource, "apiKey", config.LLM_API_KEY
|
||||
),
|
||||
vision_llm_base_url=str(
|
||||
_value(vision_resource, "apiUrl", config.LLM_BASE_URL)
|
||||
),
|
||||
stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY),
|
||||
stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)),
|
||||
tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY),
|
||||
|
||||
Reference in New Issue
Block a user