feat: implement OpenAI-compatible Realtime API with authentication and management features
- Added support for public Realtime API, including new routes for managing API keys and handling WebRTC connections. - Introduced RealtimeApiKey model and associated CRUD operations for admin management of API keys. - Implemented authentication mechanisms for API keys and client secrets. - Enhanced environment configuration with new secrets for Realtime API. - Created OpenAIRealtime session management and event processing for real-time interactions. - Updated schemas and settings to accommodate new features and ensure compatibility with existing systems.
This commit is contained in:
Xin Wang
@@ -47,6 +47,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
|
||||
from pipecat.flows import FlowsFunctionSchema
|
||||
from pipecat.frames.frames import (
|
||||
EndFrame,
|
||||
InterruptionFrame,
|
||||
LLMMessagesAppendFrame,
|
||||
OutputTransportMessageUrgentFrame,
|
||||
UserImageRawFrame,
|
||||
@@ -94,6 +95,11 @@ from services.pipecat.pipeline_events import (
|
||||
bind_cascade_pipeline_events,
|
||||
bind_realtime_pipeline_events,
|
||||
)
|
||||
from services.realtime.protocol import (
|
||||
PipelineProtocolAdapter,
|
||||
PipelineProtocolRuntime,
|
||||
RealtimeProviderControlFrame,
|
||||
)
|
||||
from pipecat.workers.runner import WorkerRunner
|
||||
|
||||
|
||||
@@ -201,6 +207,7 @@ async def run_pipeline(
|
||||
vision_enabled: bool = False,
|
||||
assistant_id: str | None = None,
|
||||
channel: str = "webrtc",
|
||||
protocol_adapter: PipelineProtocolAdapter | None = None,
|
||||
) -> None:
|
||||
"""在给定 transport 上构建并运行管线,直到连接结束。
|
||||
|
||||
@@ -234,6 +241,7 @@ async def run_pipeline(
|
||||
vision_enabled=vision_enabled,
|
||||
assistant_id=assistant_id,
|
||||
channel=channel,
|
||||
protocol_adapter=protocol_adapter,
|
||||
)
|
||||
return
|
||||
|
||||
@@ -626,15 +634,22 @@ async def run_pipeline(
|
||||
analysis_plan=cfg.analysis_config,
|
||||
extra=(workflow_engine.session_metadata() if workflow_engine else None),
|
||||
)
|
||||
protocol_inputs = protocol_adapter.input_processors() if protocol_adapter else []
|
||||
protocol_inference = (
|
||||
protocol_adapter.inference_processors() if protocol_adapter else []
|
||||
)
|
||||
protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(),
|
||||
*protocol_inputs,
|
||||
client_tools,
|
||||
session_update,
|
||||
vision_capture,
|
||||
user_input,
|
||||
stt_processor,
|
||||
user_aggregator,
|
||||
*protocol_inference,
|
||||
user_turn_router,
|
||||
knowledge_retrieval,
|
||||
llm,
|
||||
@@ -645,6 +660,7 @@ async def run_pipeline(
|
||||
tts_processor,
|
||||
EndCallAfterSpeechProcessor(call_end),
|
||||
ConversationHistoryProcessor(recorder),
|
||||
*protocol_outputs,
|
||||
transport.output(),
|
||||
]
|
||||
)
|
||||
@@ -657,6 +673,47 @@ async def run_pipeline(
|
||||
enable_rtvi=False,
|
||||
)
|
||||
worker_holder["worker"] = worker
|
||||
protocol_turn_state: dict[str, bool | None] = {
|
||||
"external": False,
|
||||
"interrupt_response": None,
|
||||
}
|
||||
if protocol_adapter:
|
||||
|
||||
async def set_external_turn_control(enabled: bool) -> None:
|
||||
protocol_turn_state["external"] = enabled
|
||||
await user_aggregator.apply_external_turn_control(enabled)
|
||||
|
||||
async def request_response() -> None:
|
||||
# Cascade inference is triggered when the adapter submits user-input.
|
||||
return None
|
||||
|
||||
async def commit_audio() -> None:
|
||||
return None
|
||||
|
||||
async def clear_audio() -> None:
|
||||
return None
|
||||
|
||||
async def set_response_interruption(enabled: bool) -> None:
|
||||
protocol_turn_state["interrupt_response"] = enabled
|
||||
await user_aggregator.apply_turn_strategies(
|
||||
cfg.turnConfig,
|
||||
enable_interruptions=enabled,
|
||||
)
|
||||
|
||||
async def cancel_response() -> None:
|
||||
await worker.queue_frame(InterruptionFrame())
|
||||
|
||||
await protocol_adapter.bind(
|
||||
PipelineProtocolRuntime(
|
||||
queue_frame=worker.queue_frame,
|
||||
set_external_turn_control=set_external_turn_control,
|
||||
set_response_interruption=set_response_interruption,
|
||||
commit_audio=commit_audio,
|
||||
clear_audio=clear_audio,
|
||||
request_response=request_response,
|
||||
cancel_response=cancel_response,
|
||||
)
|
||||
)
|
||||
service_controller = WorkflowServiceController(
|
||||
worker=worker,
|
||||
llm_services=llm_services,
|
||||
@@ -685,6 +742,15 @@ async def run_pipeline(
|
||||
normalized,
|
||||
enable_interruptions=enable_interrupt,
|
||||
)
|
||||
if protocol_turn_state["external"]:
|
||||
await user_aggregator.apply_external_turn_control(True)
|
||||
elif protocol_turn_state["interrupt_response"] is not None:
|
||||
await user_aggregator.apply_turn_strategies(
|
||||
normalized,
|
||||
enable_interruptions=bool(
|
||||
protocol_turn_state["interrupt_response"]
|
||||
),
|
||||
)
|
||||
await worker.queue_frame(
|
||||
VADParamsUpdateFrame(params=create_vad_params(normalized))
|
||||
)
|
||||
@@ -872,6 +938,7 @@ async def run_realtime_pipeline(
|
||||
vision_enabled: bool = False,
|
||||
assistant_id: str | None = None,
|
||||
channel: str = "webrtc",
|
||||
protocol_adapter: PipelineProtocolAdapter | None = None,
|
||||
) -> None:
|
||||
"""Run a speech-to-speech model that owns ASR, reasoning, and synthesis."""
|
||||
realtime = create_realtime_service(
|
||||
@@ -960,18 +1027,26 @@ async def run_realtime_pipeline(
|
||||
else None
|
||||
),
|
||||
)
|
||||
protocol_inputs = protocol_adapter.input_processors() if protocol_adapter else []
|
||||
protocol_inference = (
|
||||
protocol_adapter.inference_processors() if protocol_adapter else []
|
||||
)
|
||||
protocol_outputs = protocol_adapter.output_processors() if protocol_adapter else []
|
||||
pipeline = Pipeline(
|
||||
[
|
||||
transport.input(),
|
||||
*protocol_inputs,
|
||||
vision_capture,
|
||||
client_tools,
|
||||
session_update,
|
||||
user_input,
|
||||
input_gate,
|
||||
*protocol_inference,
|
||||
realtime,
|
||||
dynamic_variables,
|
||||
EndCallAfterSpeechProcessor(call_end),
|
||||
ConversationHistoryProcessor(recorder),
|
||||
*protocol_outputs,
|
||||
transport.output(),
|
||||
]
|
||||
)
|
||||
@@ -985,6 +1060,46 @@ async def run_realtime_pipeline(
|
||||
enable_rtvi=False,
|
||||
)
|
||||
worker_holder["worker"] = worker
|
||||
if protocol_adapter:
|
||||
|
||||
async def set_external_turn_control(enabled: bool) -> None:
|
||||
update = getattr(realtime, "update_turn_detection", None)
|
||||
if callable(update):
|
||||
await update(None if enabled else cfg.turnConfig)
|
||||
|
||||
async def request_response() -> None:
|
||||
await worker.queue_frame(
|
||||
RealtimeProviderControlFrame(action="request_response")
|
||||
)
|
||||
|
||||
async def commit_audio() -> None:
|
||||
await worker.queue_frame(
|
||||
RealtimeProviderControlFrame(action="commit_audio")
|
||||
)
|
||||
|
||||
async def clear_audio() -> None:
|
||||
await worker.queue_frame(
|
||||
RealtimeProviderControlFrame(action="clear_audio")
|
||||
)
|
||||
|
||||
async def cancel_response() -> None:
|
||||
await realtime.interrupt()
|
||||
|
||||
async def set_response_interruption(_enabled: bool) -> None:
|
||||
# Provider-side settings vary; explicit response.cancel remains portable.
|
||||
return None
|
||||
|
||||
await protocol_adapter.bind(
|
||||
PipelineProtocolRuntime(
|
||||
queue_frame=worker.queue_frame,
|
||||
set_external_turn_control=set_external_turn_control,
|
||||
set_response_interruption=set_response_interruption,
|
||||
commit_audio=commit_audio,
|
||||
clear_audio=clear_audio,
|
||||
request_response=request_response,
|
||||
cancel_response=cancel_response,
|
||||
)
|
||||
)
|
||||
|
||||
def set_input_enabled(enabled: bool) -> None:
|
||||
input_state["enabled"] = enabled
|
||||
|
||||
Reference in New Issue
Block a user