Add support for image input in model resources and enhance related configurations

- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input.
- Update the backend models, schemas, and database seed scripts to accommodate the new field.
- Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling.
- Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support.
- Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
This commit is contained in:
Xin Wang
2026-07-07 19:41:45 +08:00
parent 32a52d318a
commit c5db918830
18 changed files with 411 additions and 116 deletions

View File

@@ -24,21 +24,26 @@ from pipecat.transports.websocket.fastapi import (
from pipecat.serializers.protobuf import ProtobufFrameSerializer
def _base_params() -> dict:
def _base_params(*, video_in_enabled: bool = False) -> dict:
"""两种 transport 共享的音频参数。"""
return dict(
audio_in_enabled=True,
audio_out_enabled=True,
video_in_enabled=video_in_enabled,
# EndFrame 后默认补 2s 静音(防止收尾被截断)。我们的挂断已等到机器人
# 说完才触发,这段静音纯属空等,置 0 让结束语播完立即挂断。
audio_out_end_silence_secs=0,
)
def build_webrtc_transport(connection: SmallWebRTCConnection) -> SmallWebRTCTransport:
def build_webrtc_transport(
connection: SmallWebRTCConnection,
*,
video_in_enabled: bool = False,
) -> SmallWebRTCTransport:
return SmallWebRTCTransport(
webrtc_connection=connection,
params=TransportParams(**_base_params()),
params=TransportParams(**_base_params(video_in_enabled=video_in_enabled)),
)