Add support for image input in model resources and enhance related configurations
- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input. - Update the backend models, schemas, and database seed scripts to accommodate the new field. - Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling. - Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support. - Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
This commit is contained in:
@@ -87,6 +87,8 @@ async def _handle_offer(websocket, payload, peers):
|
||||
await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
|
||||
else:
|
||||
cfg = await _resolve_config(offer) # 解析放在建连前,配置错就别建连
|
||||
if offer.vision_enabled and not cfg.llm_support_image_input:
|
||||
raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM")
|
||||
pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
|
||||
if pc_id:
|
||||
pc._pc_id = pc_id
|
||||
@@ -98,8 +100,13 @@ async def _handle_offer(websocket, payload, peers):
|
||||
peers.pop(conn.pc_id, None)
|
||||
|
||||
# 后台跑管线:WebRTC transport + 解析出的运行时配置
|
||||
transport = build_webrtc_transport(pc)
|
||||
asyncio.create_task(run_pipeline(transport, cfg))
|
||||
transport = build_webrtc_transport(
|
||||
pc,
|
||||
video_in_enabled=offer.vision_enabled,
|
||||
)
|
||||
asyncio.create_task(
|
||||
run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled)
|
||||
)
|
||||
|
||||
answer = pc.get_answer()
|
||||
if websocket.application_state == WebSocketState.CONNECTED:
|
||||
|
||||
Reference in New Issue
Block a user