c5db918830
- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input. - Update the backend models, schemas, and database seed scripts to accommodate the new field. - Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling. - Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support. - Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
85 lines
2.6 KiB
Python
85 lines
2.6 KiB
Python
"""请求/配置数据模型。
|
|
|
|
分两层(重要):
|
|
- AssistantConfig:**运行时**配置,含真 key,只在后端内部流转,绝不返回前端。
|
|
由 config_resolver 从 DB 组装(或信令内联传入)。
|
|
- schemas.py 里的 *Request/*Response:面向前端的 DTO(key 打码)。
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Literal
|
|
|
|
from pydantic import BaseModel, Field
|
|
|
|
RuntimeMode = Literal["pipeline", "realtime"]
|
|
|
|
|
|
class AssistantConfig(BaseModel):
|
|
"""运行时配置:前端可见部分(name/prompt/...) + 服务端注入部分(*_api_key/*_base_url)。"""
|
|
|
|
name: str = "未命名助手"
|
|
# prompt|workflow|dify|fastgpt|opencode;决定由哪种「大脑」驱动对话
|
|
type: str = "prompt"
|
|
greeting: str = "您好,我是 AI 视频助手,请问有什么可以帮您?"
|
|
prompt: str = "你是一个有帮助的助手。"
|
|
runtimeMode: RuntimeMode = "pipeline"
|
|
|
|
# 模型/音色选项
|
|
model: str = "" # LLM
|
|
asr: str = "" # STT
|
|
tts_model: str = ""
|
|
voice: str = "" # TTS 音色
|
|
stt_language: str = ""
|
|
tts_speed: float = 1.0
|
|
realtimeModel: str = ""
|
|
realtime_interface_type: str = ""
|
|
realtime_values: dict = {}
|
|
realtime_secrets: dict = {}
|
|
llm_interface_type: str = "openai-llm"
|
|
stt_interface_type: str = "openai-asr"
|
|
tts_interface_type: str = "openai-tts"
|
|
llm_values: dict = {}
|
|
llm_secrets: dict = {}
|
|
llm_support_image_input: bool = False
|
|
stt_values: dict = {}
|
|
stt_secrets: dict = {}
|
|
tts_values: dict = {}
|
|
tts_secrets: dict = {}
|
|
|
|
enableInterrupt: bool = True
|
|
|
|
# workflow 类型:节点图(nodes/edges)。非 workflow 为空,引擎据此决定是否启用。
|
|
graph: dict = {}
|
|
|
|
# 外部托管类型(fastgpt/dify/opencode)的连接信息:context/KB/tools 由对方服务端接管。
|
|
fastgpt_api_url: str = ""
|
|
fastgpt_api_key: str = ""
|
|
fastgpt_app_id: str = ""
|
|
|
|
# ---- 运行时连接信息(服务端注入,不来自浏览器) ----
|
|
# 为空时,service_factory 会回退到 config.py 的 .env 默认值。
|
|
llm_api_key: str = ""
|
|
llm_base_url: str = ""
|
|
stt_api_key: str = ""
|
|
stt_base_url: str = ""
|
|
tts_api_key: str = ""
|
|
tts_base_url: str = ""
|
|
realtime_api_key: str = ""
|
|
realtime_base_url: str = ""
|
|
|
|
|
|
class SignalingOffer(BaseModel):
|
|
"""WS 信令里 offer 消息的 payload。
|
|
|
|
推荐用 assistant_id(浏览器只传 id,key 在服务端解析);
|
|
inline_config 仅用于调试/无库场景。
|
|
"""
|
|
|
|
pc_id: str | None = None
|
|
sdp: str
|
|
type: str
|
|
assistant_id: str | None = None
|
|
inline_config: AssistantConfig | None = None
|
|
vision_enabled: bool = False
|