86639692ba
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections. - Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys. - Implemented authentication mechanisms for API keys and client secrets. - Enhanced environment configuration with new secrets for Realtime API. - Created OpenAIRealtime session management and event processing for real-time interactions. - Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
342 lines
13 KiB
Python
342 lines
13 KiB
Python
"""Public OpenAI-compatible Realtime HTTP, WebRTC, and WebSocket entries."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from db.session import SessionLocal
|
|
from fastapi import APIRouter, Request, WebSocket
|
|
from fastapi.responses import JSONResponse, Response
|
|
from loguru import logger
|
|
from starlette.websockets import WebSocketState
|
|
|
|
from services.openai_realtime.auth import (
|
|
RealtimeAuthError,
|
|
RealtimeCredential,
|
|
authenticate_bearer,
|
|
bearer_from_authorization,
|
|
create_client_secret,
|
|
hash_safety_identifier,
|
|
)
|
|
from services.openai_realtime.bridge import OpenAIRealtimeBridge
|
|
from services.openai_realtime.events import (
|
|
RealtimeEventError,
|
|
assistant_id_from_model,
|
|
)
|
|
from services.openai_realtime.session import OpenAIRealtimeSession
|
|
from services.openai_realtime.webrtc import OpenAIRealtimeWebRTCConnection
|
|
from services.openai_realtime.websocket import build_openai_websocket_transport
|
|
from services.pipecat.pipeline import run_pipeline
|
|
from services.pipecat.transports import build_webrtc_transport
|
|
from services.realtime import lifecycle
|
|
from services.realtime.launcher import (
|
|
resolve_assistant_config,
|
|
validate_runtime_requirements,
|
|
validate_visual_runtime,
|
|
)
|
|
from services.webrtc_ice import aiortc_ice_servers
|
|
|
|
|
|
router = APIRouter(prefix="/v1/realtime", tags=["openai-realtime"])
|
|
_webrtc_peers: dict[str, OpenAIRealtimeWebRTCConnection] = {}
|
|
|
|
|
|
def _api_error(message: str, *, code: str, status_code: int) -> JSONResponse:
|
|
return JSONResponse(
|
|
status_code=status_code,
|
|
content={
|
|
"error": {
|
|
"type": "invalid_request_error",
|
|
"code": code,
|
|
"message": message,
|
|
"param": None,
|
|
}
|
|
},
|
|
)
|
|
|
|
|
|
async def _credential(request: Request) -> RealtimeCredential:
|
|
token = bearer_from_authorization(request.headers.get("authorization"))
|
|
async with SessionLocal() as db:
|
|
return await authenticate_bearer(db, token)
|
|
|
|
|
|
def _session_payload(value: object) -> dict[str, Any]:
|
|
if not isinstance(value, dict):
|
|
raise RealtimeEventError("session must be an object", param="session")
|
|
session = value.get("session", value)
|
|
if not isinstance(session, dict):
|
|
raise RealtimeEventError("session must be an object", param="session")
|
|
return session
|
|
|
|
|
|
def _reject_locked_creation_fields(session: dict[str, Any]) -> None:
|
|
locked = {"instructions", "voice", "tools", "tool_choice"}.intersection(session)
|
|
audio = session.get("audio")
|
|
if isinstance(audio, dict) and isinstance(audio.get("output"), dict):
|
|
if "voice" in audio["output"]:
|
|
locked.add("audio.output.voice")
|
|
if locked:
|
|
field = sorted(locked)[0]
|
|
raise RealtimeEventError(
|
|
f"{field} is owned by the selected assistant",
|
|
code="immutable_session_field",
|
|
param=f"session.{field}",
|
|
)
|
|
modalities = session.get("output_modalities")
|
|
if modalities is not None and modalities not in (["audio"], ["text"]):
|
|
raise RealtimeEventError(
|
|
'output_modalities must be ["audio"] or ["text"]',
|
|
param="session.output_modalities",
|
|
)
|
|
|
|
|
|
async def _build_session(
|
|
credential: RealtimeCredential,
|
|
requested: dict[str, Any] | None,
|
|
*,
|
|
safety_identifier_hash: str | None,
|
|
) -> OpenAIRealtimeSession:
|
|
session_value = dict(credential.session or requested or {})
|
|
_reject_locked_creation_fields(session_value)
|
|
requested_assistant_id = assistant_id_from_model(session_value.get("model"))
|
|
if credential.assistant_id and requested_assistant_id != credential.assistant_id:
|
|
raise RealtimeEventError(
|
|
"Client secret is bound to another assistant",
|
|
code="invalid_model",
|
|
param="session.model",
|
|
)
|
|
config = await resolve_assistant_config(requested_assistant_id)
|
|
validate_runtime_requirements(config)
|
|
vision_enabled = validate_visual_runtime(config)
|
|
state = OpenAIRealtimeSession(
|
|
assistant_id=requested_assistant_id,
|
|
config=config,
|
|
vision_enabled=vision_enabled,
|
|
safety_identifier_hash=(
|
|
credential.safety_identifier_hash or safety_identifier_hash
|
|
),
|
|
)
|
|
state.apply_initial_options(session_value)
|
|
return state
|
|
|
|
|
|
@router.post("/client_secrets")
|
|
async def create_realtime_client_secret(request: Request):
|
|
try:
|
|
credential = await _credential(request)
|
|
if credential.ephemeral:
|
|
raise RealtimeAuthError("A long-lived API key is required")
|
|
body = await request.json()
|
|
session = _session_payload(body)
|
|
_reject_locked_creation_fields(session)
|
|
assistant_id = assistant_id_from_model(session.get("model"))
|
|
# Resolve now so invalid models/configurations fail before a browser gets a token.
|
|
config = await resolve_assistant_config(assistant_id)
|
|
validate_runtime_requirements(config)
|
|
validate_visual_runtime(config)
|
|
safety_hash = hash_safety_identifier(
|
|
request.headers.get("openai-safety-identifier")
|
|
)
|
|
value, expires_at = create_client_secret(
|
|
api_key_id=credential.api_key_id,
|
|
assistant_id=assistant_id,
|
|
session=session,
|
|
safety_identifier_hash=safety_hash,
|
|
)
|
|
return {
|
|
"value": value,
|
|
"expires_at": expires_at,
|
|
"session": session,
|
|
}
|
|
except RealtimeAuthError as exc:
|
|
return _api_error(str(exc), code="invalid_api_key", status_code=401)
|
|
except RealtimeEventError as exc:
|
|
return _api_error(str(exc), code=exc.code, status_code=400)
|
|
except ValueError as exc:
|
|
code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session"
|
|
return _api_error(str(exc), code=code, status_code=400)
|
|
|
|
|
|
@router.post("/calls")
|
|
async def create_realtime_call(request: Request):
|
|
try:
|
|
credential = await _credential(request)
|
|
content_type = request.headers.get("content-type", "").lower()
|
|
if content_type.startswith("multipart/form-data"):
|
|
if credential.ephemeral:
|
|
raise RealtimeAuthError(
|
|
"Client secrets must send an application/sdp offer"
|
|
)
|
|
form = await request.form()
|
|
sdp = str(form.get("sdp") or "")
|
|
raw_session = str(form.get("session") or "{}")
|
|
requested = _session_payload(json.loads(raw_session))
|
|
elif content_type.startswith("application/sdp"):
|
|
if not credential.ephemeral:
|
|
raise RealtimeAuthError(
|
|
"application/sdp requires a short-lived client secret"
|
|
)
|
|
sdp = (await request.body()).decode("utf-8")
|
|
requested = None
|
|
else:
|
|
raise RealtimeEventError(
|
|
"Use multipart/form-data or application/sdp",
|
|
code="unsupported_content_type",
|
|
)
|
|
if not sdp.strip():
|
|
raise RealtimeEventError("SDP offer is empty", param="sdp")
|
|
state = await _build_session(
|
|
credential,
|
|
requested,
|
|
safety_identifier_hash=hash_safety_identifier(
|
|
request.headers.get("openai-safety-identifier")
|
|
),
|
|
)
|
|
answer = await _start_webrtc(sdp, state)
|
|
return Response(
|
|
content=answer,
|
|
media_type="application/sdp",
|
|
headers={"Location": f"/v1/realtime/calls/{state.id}"},
|
|
)
|
|
except RealtimeAuthError as exc:
|
|
return _api_error(str(exc), code="invalid_api_key", status_code=401)
|
|
except (RealtimeEventError, json.JSONDecodeError, UnicodeDecodeError) as exc:
|
|
code = exc.code if isinstance(exc, RealtimeEventError) else "invalid_request_error"
|
|
return _api_error(str(exc), code=code, status_code=400)
|
|
except ValueError as exc:
|
|
code = "invalid_model" if "助手不存在" in str(exc) else "invalid_session"
|
|
return _api_error(str(exc), code=code, status_code=400)
|
|
except Exception as exc: # noqa: BLE001 - no internal details in public response
|
|
logger.exception(f"OpenAI Realtime WebRTC 启动失败: {exc}")
|
|
return _api_error(
|
|
"Realtime connection could not be established",
|
|
code="connection_error",
|
|
status_code=500,
|
|
)
|
|
|
|
|
|
async def _start_webrtc(sdp: str, state: OpenAIRealtimeSession) -> str:
|
|
connection = OpenAIRealtimeWebRTCConnection(
|
|
ice_servers=aiortc_ice_servers()
|
|
)
|
|
await connection.initialize(sdp=sdp, type="offer")
|
|
_webrtc_peers[connection.pc_id] = connection
|
|
bridge = OpenAIRealtimeBridge(state, channel="webrtc")
|
|
transport = build_webrtc_transport(
|
|
connection,
|
|
video_in_enabled=state.vision_enabled,
|
|
)
|
|
task = lifecycle.start_pipeline_task(
|
|
connection,
|
|
run_pipeline(
|
|
transport,
|
|
state.config,
|
|
vision_enabled=state.vision_enabled,
|
|
assistant_id=state.assistant_id,
|
|
channel="openai-webrtc",
|
|
protocol_adapter=bridge,
|
|
),
|
|
protocol="openai-webrtc",
|
|
)
|
|
|
|
@connection.event_handler("closed")
|
|
async def on_closed(conn: OpenAIRealtimeWebRTCConnection):
|
|
_webrtc_peers.pop(conn.pc_id, None)
|
|
lifecycle.active_connections.discard(conn)
|
|
await lifecycle.wait_for_pipeline_close(
|
|
task,
|
|
connection_id=conn.pc_id,
|
|
)
|
|
|
|
answer = connection.get_answer()
|
|
if not answer:
|
|
raise RuntimeError("WebRTC answer was not created")
|
|
return str(answer["sdp"])
|
|
|
|
|
|
def _websocket_token(websocket: WebSocket) -> tuple[str, str | None]:
|
|
authorization = websocket.headers.get("authorization")
|
|
if authorization:
|
|
return bearer_from_authorization(authorization), None
|
|
protocols = [
|
|
item.strip()
|
|
for item in websocket.headers.get("sec-websocket-protocol", "").split(",")
|
|
if item.strip()
|
|
]
|
|
for protocol in protocols:
|
|
prefix = "openai-insecure-api-key."
|
|
if protocol.startswith(prefix):
|
|
return protocol.removeprefix(prefix), "realtime" if "realtime" in protocols else None
|
|
raise RealtimeAuthError("Missing Bearer credential")
|
|
|
|
|
|
@dataclass(eq=False)
|
|
class _ManagedWebSocket:
|
|
websocket: WebSocket
|
|
pc_id: str = field(default_factory=lambda: f"ws_{uuid4().hex}")
|
|
|
|
async def disconnect(self) -> None:
|
|
if self.websocket.application_state == WebSocketState.CONNECTED:
|
|
await self.websocket.close(code=1001)
|
|
|
|
|
|
@router.websocket("")
|
|
async def realtime_websocket(websocket: WebSocket):
|
|
managed: _ManagedWebSocket | None = None
|
|
task = None
|
|
try:
|
|
token, accepted_subprotocol = _websocket_token(websocket)
|
|
async with SessionLocal() as db:
|
|
credential = await authenticate_bearer(db, token)
|
|
requested = (
|
|
credential.session
|
|
if credential.ephemeral
|
|
else {"model": websocket.query_params.get("model")}
|
|
)
|
|
state = await _build_session(
|
|
credential,
|
|
requested,
|
|
safety_identifier_hash=hash_safety_identifier(
|
|
websocket.headers.get("openai-safety-identifier")
|
|
),
|
|
)
|
|
await websocket.accept(subprotocol=accepted_subprotocol)
|
|
managed = _ManagedWebSocket(websocket)
|
|
transport = build_openai_websocket_transport(websocket)
|
|
bridge = OpenAIRealtimeBridge(state, channel="websocket")
|
|
task = lifecycle.start_pipeline_task(
|
|
managed,
|
|
run_pipeline(
|
|
transport,
|
|
state.config,
|
|
vision_enabled=False,
|
|
assistant_id=state.assistant_id,
|
|
channel="openai-websocket",
|
|
protocol_adapter=bridge,
|
|
),
|
|
protocol="openai-websocket",
|
|
)
|
|
await task
|
|
except (RealtimeAuthError, RealtimeEventError, ValueError) as exc:
|
|
logger.warning(f"拒绝 OpenAI Realtime WebSocket: {exc}")
|
|
if websocket.application_state == WebSocketState.CONNECTED:
|
|
await websocket.close(code=1008, reason=str(exc)[:120])
|
|
else:
|
|
await websocket.close(code=1008)
|
|
except Exception as exc: # noqa: BLE001 - pipeline callback logs full exception
|
|
logger.warning(f"OpenAI Realtime WebSocket 已关闭: {type(exc).__name__}")
|
|
if websocket.application_state == WebSocketState.CONNECTED:
|
|
await websocket.close(code=1011)
|
|
finally:
|
|
if managed:
|
|
lifecycle.active_connections.discard(managed)
|
|
if task and not task.done():
|
|
await lifecycle.wait_for_pipeline_close(
|
|
task,
|
|
connection_id=managed.pc_id if managed else "websocket",
|
|
)
|