Add support for image input in model resources and enhance related configurations
- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input. - Update the backend models, schemas, and database seed scripts to accommodate the new field. - Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling. - Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support. - Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
This commit is contained in:
@@ -24,21 +24,26 @@ from pipecat.transports.websocket.fastapi import (
|
||||
from pipecat.serializers.protobuf import ProtobufFrameSerializer
|
||||
|
||||
|
||||
def _base_params() -> dict:
|
||||
def _base_params(*, video_in_enabled: bool = False) -> dict:
|
||||
"""两种 transport 共享的音频参数。"""
|
||||
return dict(
|
||||
audio_in_enabled=True,
|
||||
audio_out_enabled=True,
|
||||
video_in_enabled=video_in_enabled,
|
||||
# EndFrame 后默认补 2s 静音(防止收尾被截断)。我们的挂断已等到机器人
|
||||
# 说完才触发,这段静音纯属空等,置 0 让结束语播完立即挂断。
|
||||
audio_out_end_silence_secs=0,
|
||||
)
|
||||
|
||||
|
||||
def build_webrtc_transport(connection: SmallWebRTCConnection) -> SmallWebRTCTransport:
|
||||
def build_webrtc_transport(
|
||||
connection: SmallWebRTCConnection,
|
||||
*,
|
||||
video_in_enabled: bool = False,
|
||||
) -> SmallWebRTCTransport:
|
||||
return SmallWebRTCTransport(
|
||||
webrtc_connection=connection,
|
||||
params=TransportParams(**_base_params()),
|
||||
params=TransportParams(**_base_params(video_in_enabled=video_in_enabled)),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user