c5db918830
- Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input. - Update the backend models, schemas, and database seed scripts to accommodate the new field. - Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling. - Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support. - Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities.
61 lines
2.1 KiB
Python
61 lines
2.1 KiB
Python
"""Transport 工厂——管线与"输出方式"解耦的关键。
|
|
|
|
同一条 STT→LLM→TTS 管线,可以挂在不同 transport 上:
|
|
- WebRTC:浏览器,低延迟,带 NAT 穿透 -> build_webrtc_transport
|
|
- WS: 裸音频流,服务端/话务/自定义客户端,简单 -> build_ws_transport
|
|
|
|
未来加电话(Twilio/Vonage)只是再加一个 build_xxx_transport + 对应 serializer。
|
|
对应 dograh 的 transport_setup.py(WebRTC)+ 各 telephony provider 的 transport.py(WS)。
|
|
"""
|
|
|
|
from fastapi import WebSocket
|
|
|
|
from pipecat.transports.base_transport import TransportParams
|
|
|
|
# WebRTC
|
|
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
|
|
from pipecat.transports.smallwebrtc.transport import SmallWebRTCTransport
|
|
|
|
# 裸 WS 音频流
|
|
from pipecat.transports.websocket.fastapi import (
|
|
FastAPIWebsocketTransport,
|
|
FastAPIWebsocketParams,
|
|
)
|
|
from pipecat.serializers.protobuf import ProtobufFrameSerializer
|
|
|
|
|
|
def _base_params(*, video_in_enabled: bool = False) -> dict:
|
|
"""两种 transport 共享的音频参数。"""
|
|
return dict(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
video_in_enabled=video_in_enabled,
|
|
# EndFrame 后默认补 2s 静音(防止收尾被截断)。我们的挂断已等到机器人
|
|
# 说完才触发,这段静音纯属空等,置 0 让结束语播完立即挂断。
|
|
audio_out_end_silence_secs=0,
|
|
)
|
|
|
|
|
|
def build_webrtc_transport(
|
|
connection: SmallWebRTCConnection,
|
|
*,
|
|
video_in_enabled: bool = False,
|
|
) -> SmallWebRTCTransport:
|
|
return SmallWebRTCTransport(
|
|
webrtc_connection=connection,
|
|
params=TransportParams(**_base_params(video_in_enabled=video_in_enabled)),
|
|
)
|
|
|
|
|
|
def build_ws_transport(websocket: WebSocket) -> FastAPIWebsocketTransport:
|
|
"""裸 WS 输出。序列化用 protobuf(自定义客户端用同款解码);
|
|
若对接电话商,把 serializer 换成对应的 TwilioFrameSerializer 等即可。
|
|
"""
|
|
return FastAPIWebsocketTransport(
|
|
websocket=websocket,
|
|
params=FastAPIWebsocketParams(
|
|
serializer=ProtobufFrameSerializer(),
|
|
**_base_params(),
|
|
),
|
|
)
|