86639692ba
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections. - Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys. - Implemented authentication mechanisms for API keys and client secrets. - Enhanced environment configuration with new secrets for Realtime API. - Created OpenAIRealtime session management and event processing for real-time interactions. - Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
142 lines
4.9 KiB
Python
142 lines
4.9 KiB
Python
"""Connection-local state for the OpenAI-compatible Realtime wire protocol."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from models import AssistantConfig
|
|
from services.openai_realtime.events import normalize_turn_detection
|
|
|
|
|
|
def _id(prefix: str) -> str:
|
|
return f"{prefix}_{uuid4().hex}"
|
|
|
|
|
|
@dataclass
|
|
class OpenAIRealtimeSession:
|
|
assistant_id: str
|
|
config: AssistantConfig
|
|
vision_enabled: bool
|
|
safety_identifier_hash: str | None = None
|
|
id: str = field(default_factory=lambda: _id("sess"))
|
|
output_modalities: list[str] = field(default_factory=lambda: ["audio"])
|
|
turn_detection: dict[str, Any] | None = field(
|
|
default_factory=lambda: {
|
|
"type": "server_vad",
|
|
"threshold": 0.7,
|
|
"prefix_padding_ms": 200,
|
|
"silence_duration_ms": 600,
|
|
"create_response": True,
|
|
"interrupt_response": True,
|
|
}
|
|
)
|
|
capabilities: set[str] = field(default_factory=set)
|
|
pending_item: dict[str, Any] | None = None
|
|
audio_chunks: list[tuple[bytes, int, int]] = field(default_factory=list)
|
|
buffered_audio_bytes: int = 0
|
|
audio_committed: bool = False
|
|
active_response_id: str | None = None
|
|
active_output_item_id: str | None = None
|
|
active_input_item_id: str | None = None
|
|
response_text: str = ""
|
|
|
|
@property
|
|
def model(self) -> str:
|
|
return f"assistant:{self.assistant_id}"
|
|
|
|
@property
|
|
def external_turn_control(self) -> bool:
|
|
return self.turn_detection is None
|
|
|
|
@property
|
|
def output_is_audio(self) -> bool:
|
|
return self.output_modalities == ["audio"]
|
|
|
|
def client_tools(self) -> list[dict[str, Any]]:
|
|
result = []
|
|
for tool in self.config.tools:
|
|
if tool.type != "client":
|
|
continue
|
|
definition = tool.definition or {}
|
|
parameters = (definition.get("config") or {}).get("parameters") or []
|
|
properties: dict[str, Any] = {}
|
|
required: list[str] = []
|
|
for parameter in parameters:
|
|
name = str(parameter.get("name") or "")
|
|
if not name:
|
|
continue
|
|
properties[name] = {
|
|
"type": parameter.get("type") or "string",
|
|
**(
|
|
{"description": parameter["description"]}
|
|
if parameter.get("description")
|
|
else {}
|
|
),
|
|
}
|
|
if parameter.get("required"):
|
|
required.append(name)
|
|
result.append(
|
|
{
|
|
"type": "function",
|
|
"name": tool.function_name,
|
|
"description": tool.description,
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": properties,
|
|
"required": required,
|
|
},
|
|
}
|
|
)
|
|
return result
|
|
|
|
def public_value(self) -> dict[str, Any]:
|
|
return {
|
|
"id": self.id,
|
|
"type": "realtime",
|
|
"model": self.model,
|
|
"output_modalities": self.output_modalities,
|
|
"audio": {
|
|
"input": {
|
|
"format": {"type": "audio/pcm", "rate": 24000},
|
|
"turn_detection": self.turn_detection,
|
|
},
|
|
"output": {
|
|
"format": {"type": "audio/pcm", "rate": 24000},
|
|
},
|
|
},
|
|
"tools": self.client_tools(),
|
|
"tool_choice": "auto",
|
|
}
|
|
|
|
def apply_initial_options(self, value: dict[str, Any]) -> None:
|
|
modalities = value.get("output_modalities")
|
|
if modalities in (["audio"], ["text"]):
|
|
self.output_modalities = list(modalities)
|
|
audio = value.get("audio")
|
|
if isinstance(audio, dict) and isinstance(audio.get("input"), dict):
|
|
turn_detection = audio["input"].get("turn_detection", self.turn_detection)
|
|
else:
|
|
turn_detection = value.get("turn_detection", self.turn_detection)
|
|
self.turn_detection = normalize_turn_detection(turn_detection)
|
|
|
|
def begin_response(self) -> tuple[str, str]:
|
|
if not self.active_response_id:
|
|
self.active_response_id = _id("resp")
|
|
if not self.active_output_item_id:
|
|
self.active_output_item_id = _id("item")
|
|
self.response_text = ""
|
|
return self.active_response_id, str(self.active_output_item_id)
|
|
|
|
def finish_response(self) -> tuple[str | None, str | None, str]:
|
|
result = (
|
|
self.active_response_id,
|
|
self.active_output_item_id,
|
|
self.response_text,
|
|
)
|
|
self.active_response_id = None
|
|
self.active_output_item_id = None
|
|
self.response_text = ""
|
|
return result
|