86639692ba
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections. - Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys. - Implemented authentication mechanisms for API keys and client secrets. - Enhanced environment configuration with new secrets for Realtime API. - Created OpenAIRealtime session management and event processing for real-time interactions. - Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
137 lines
4.1 KiB
Python
137 lines
4.1 KiB
Python
"""Transport 工厂——管线与"输出方式"解耦的关键。
|
|
|
|
同一条 STT→LLM→TTS 管线,可以挂在不同 transport 上:
|
|
- WebRTC:浏览器,低延迟,带 NAT 穿透 -> build_webrtc_transport
|
|
- WS: 裸音频流,服务端/话务/自定义客户端,简单 -> build_ws_transport
|
|
|
|
未来加电话(Twilio/Vonage)只是再加一个 build_xxx_transport + 对应 serializer。
|
|
对应 dograh 的 transport_setup.py(WebRTC)+ 各 telephony provider 的 transport.py(WS)。
|
|
"""
|
|
|
|
from fastapi import WebSocket
|
|
|
|
from pipecat.transports.base_transport import TransportParams
|
|
|
|
# WebRTC
|
|
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
|
|
from pipecat.transports.smallwebrtc.transport import (
|
|
SmallWebRTCOutputTransport,
|
|
SmallWebRTCTransport,
|
|
)
|
|
|
|
# 裸 WS 音频流
|
|
from pipecat.transports.websocket.fastapi import (
|
|
FastAPIWebsocketOutputTransport,
|
|
FastAPIWebsocketTransport,
|
|
FastAPIWebsocketParams,
|
|
)
|
|
from pipecat.serializers.base_serializer import FrameSerializer
|
|
from pipecat.serializers.protobuf import ProtobufFrameSerializer
|
|
|
|
from services.pipecat.call_lifecycle import FixedSpeechPlaybackMarkerFrame
|
|
|
|
|
|
class _PlaybackMarkerOutputMixin:
|
|
"""Resolve fixed-speech markers after preceding audio has been sent."""
|
|
|
|
async def write_transport_frame(self, frame):
|
|
if isinstance(frame, FixedSpeechPlaybackMarkerFrame):
|
|
await frame.completion.mark_played()
|
|
return
|
|
await super().write_transport_frame(frame)
|
|
|
|
|
|
class _WebRTCOutputTransport(
|
|
_PlaybackMarkerOutputMixin,
|
|
SmallWebRTCOutputTransport,
|
|
):
|
|
pass
|
|
|
|
|
|
class _WebRTCTransport(SmallWebRTCTransport):
|
|
def output(self) -> SmallWebRTCOutputTransport:
|
|
if not self._output:
|
|
self._output = _WebRTCOutputTransport(
|
|
self._client,
|
|
self._params,
|
|
name=self._input_name,
|
|
)
|
|
return self._output
|
|
|
|
|
|
class _WebsocketOutputTransport(
|
|
_PlaybackMarkerOutputMixin,
|
|
FastAPIWebsocketOutputTransport,
|
|
):
|
|
pass
|
|
|
|
|
|
class _WebsocketTransport(FastAPIWebsocketTransport):
|
|
def __init__(self, websocket: WebSocket, params: FastAPIWebsocketParams):
|
|
super().__init__(websocket=websocket, params=params)
|
|
self._output = _WebsocketOutputTransport(
|
|
self,
|
|
self._client,
|
|
self._params,
|
|
name=self._output_name,
|
|
)
|
|
|
|
|
|
def _base_params(*, video_in_enabled: bool = False) -> dict:
|
|
"""两种 transport 共享的音频参数。"""
|
|
return dict(
|
|
audio_in_enabled=True,
|
|
audio_out_enabled=True,
|
|
video_in_enabled=video_in_enabled,
|
|
# EndFrame 后默认补 2s 静音(防止收尾被截断)。我们的挂断已等到机器人
|
|
# 说完才触发,这段静音纯属空等,置 0 让结束语播完立即挂断。
|
|
audio_out_end_silence_secs=0,
|
|
)
|
|
|
|
|
|
def build_webrtc_transport(
|
|
connection: SmallWebRTCConnection,
|
|
*,
|
|
video_in_enabled: bool = False,
|
|
) -> SmallWebRTCTransport:
|
|
return _WebRTCTransport(
|
|
webrtc_connection=connection,
|
|
params=TransportParams(**_base_params(video_in_enabled=video_in_enabled)),
|
|
)
|
|
|
|
|
|
def build_ws_transport(websocket: WebSocket) -> FastAPIWebsocketTransport:
|
|
"""裸 WS 输出。序列化用 protobuf(自定义客户端用同款解码);
|
|
若对接电话商,把 serializer 换成对应的 TwilioFrameSerializer 等即可。
|
|
"""
|
|
return build_serialized_ws_transport(
|
|
websocket,
|
|
serializer=ProtobufFrameSerializer(),
|
|
)
|
|
|
|
|
|
def build_serialized_ws_transport(
|
|
websocket: WebSocket,
|
|
*,
|
|
serializer: FrameSerializer,
|
|
sample_rate: int | None = None,
|
|
) -> FastAPIWebsocketTransport:
|
|
"""Build a text/binary WS transport without coupling it to one protocol."""
|
|
|
|
sample_rates = (
|
|
{
|
|
"audio_in_sample_rate": sample_rate,
|
|
"audio_out_sample_rate": sample_rate,
|
|
}
|
|
if sample_rate
|
|
else {}
|
|
)
|
|
return _WebsocketTransport(
|
|
websocket=websocket,
|
|
params=FastAPIWebsocketParams(
|
|
serializer=serializer,
|
|
**_base_params(),
|
|
**sample_rates,
|
|
),
|
|
)
|