wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(workflow): add edge-tool routing and realtime runtime

Browse Source
This commit is contained in:
Xin Wang
2026-08-04 22:29:04 +08:00
parent 1902ffc240
commit 59588ba88d
25 changed files with 1952 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
backend/services/pipecat/pipeline.py
View File

@@ -16,6 +16,7 @@ from models import AssistantConfig
from openai import AsyncOpenAI
from PIL import Image
from services.brains import Brain, BrainRuntime, build_brain
from services.brains.base import RealtimeBrainRuntime
from services.conversation_history import ConversationRecorder
from services.pipecat.call_lifecycle import (
CallEndCoordinator,
@@ -73,6 +74,7 @@ from services.pipecat.processors import (
KnowledgeRetrievalProcessor,
PassthroughLLMAssistantAggregator,
RealtimeDynamicVariableProcessor,
RealtimeInputAudioGateProcessor,
RealtimeUserInputProcessor,
SessionUpdateProcessor,
UserInput,
@@ -890,7 +892,32 @@ async def run_realtime_pipeline(
instructions=brain.system_prompt(cfg),
)
input_sample_rate, output_sample_rate = realtime_audio_sample_rates(cfg)
user_input = RealtimeUserInputProcessor()
worker_holder: dict[str, PipelineWorker] = {}
input_state = {"enabled": True}
async def queue_call_end(reason: str) -> None:
worker = worker_holder.get("worker")
if worker is None:
return
logger.info(f"结束 Realtime 通话: reason={reason}")
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message={"type": "call-ended", "reason": reason}
)
)
await worker.queue_frame(EndFrame())
call_end = CallEndCoordinator(queue_call_end)
client_tools = ClientToolBroker()
client_tools.set_interrupt_handler(realtime.interrupt)
user_input = RealtimeUserInputProcessor(
should_ignore_input=lambda: (
call_end.ending or not input_state["enabled"]
)
)
input_gate = RealtimeInputAudioGateProcessor(
lambda: not call_end.ending and input_state["enabled"]
)
dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime)
async def refresh_realtime_instructions() -> None:
@@ -910,14 +937,22 @@ async def run_realtime_pipeline(
channel=channel,
runtime_mode=cfg.runtimeMode,
session_id=cfg.conversation_id or None,
extra=(
WorkflowEngine(cfg.graph).session_metadata()
if cfg.type == "workflow"
else None
),
)
pipeline = Pipeline(
[
transport.input(),
client_tools,
session_update,
user_input,
input_gate,
realtime,
dynamic_variables,
EndCallAfterSpeechProcessor(call_end),
ConversationHistoryProcessor(recorder),
transport.output(),
]
@@ -931,11 +966,28 @@ async def run_realtime_pipeline(
),
enable_rtvi=False,
)
worker_holder["worker"] = worker
def set_input_enabled(enabled: bool) -> None:
input_state["enabled"] = enabled
await brain.setup_realtime(
cfg,
RealtimeBrainRuntime(
realtime=realtime,
queue_frame=worker.queue_frame,
call_end=call_end,
session_id=cfg.conversation_id or "",
client_tools=client_tools,
set_input_enabled=set_input_enabled,
),
)
bind_realtime_pipeline_events(
transport=transport,
worker=worker,
realtime=realtime,
brain=brain,
text_input=user_input,
greeting=greeting,
)

15
backend/services/pipecat/pipeline_events.py
View File

@@ -227,6 +227,7 @@ def bind_realtime_pipeline_events(
transport,
worker,
realtime,
brain,
text_input,
greeting: str,
) -> None:
@@ -251,10 +252,16 @@ def bind_realtime_pipeline_events(
await queue_transcript("user", user_input.text)
if user_input.run_immediately and user_input.interrupt:
await realtime.interrupt()
await realtime.send_text(
user_input.text,
run_immediately=user_input.run_immediately,
handled = (
await brain.on_user_turn_end(user_input.text)
if brain.spec.type == "workflow"
else False
)
if not handled:
await realtime.send_text(
user_input.text,
run_immediately=user_input.run_immediately,
)
await worker.queue_frame(
OutputTransportMessageUrgentFrame(
message={
@@ -267,6 +274,8 @@ def bind_realtime_pipeline_events(
@transport.event_handler("on_client_connected")
async def on_client_connected(_transport, _client):
await brain.on_connected(greeting_pending=bool(greeting))
await brain.on_client_ready()
if greeting:
await realtime.speak(greeting)

24
backend/services/pipecat/processors.py
View File

@@ -19,6 +19,7 @@ from pipecat.frames.frames import (
FunctionCallCancelFrame,
FunctionCallResultFrame,
FunctionCallsStartedFrame,
InputAudioRawFrame,
InputTransportMessageFrame,
InterruptionFrame,
LLMContextFrame,
@@ -520,11 +521,26 @@ class RealtimeDynamicVariableProcessor(FrameProcessor):
await self.push_frame(frame, direction)
class RealtimeInputAudioGateProcessor(FrameProcessor):
"""Drop live microphone frames while a deterministic node owns the turn."""
def __init__(self, is_enabled: Callable[[], bool]):
super().__init__()
self._is_enabled = is_enabled
async def process_frame(self, frame, direction: FrameDirection):
await super().process_frame(frame, direction)
if isinstance(frame, InputAudioRawFrame) and not self._is_enabled():
return
await self.push_frame(frame, direction)
class RealtimeUserInputProcessor(FrameProcessor):
"""Route text-only user-input messages to a realtime service."""
def __init__(self):
def __init__(self, should_ignore_input: Callable[[], bool] | None = None):
super().__init__()
self._should_ignore_input = should_ignore_input or (lambda: False)
self._register_event_handler("on_user_input")
async def process_frame(self, frame, direction: FrameDirection):
@@ -542,6 +558,12 @@ class RealtimeUserInputProcessor(FrameProcessor):
if user_input is None:
await self.push_frame(frame, direction)
return
if self._should_ignore_input():
await self._emit_error(
user_input.input_id,
"当前工作流节点暂不接收用户输入",
)
return
if user_input.has_camera_frame:
await self._emit_error(
user_input.input_id,

135
backend/services/pipecat/qwen_audio_realtime.py
View File

@@ -35,6 +35,12 @@ from pipecat.utils.time import time_now_iso8601
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
from services.pipecat.realtime_tools import (
RealtimeTool,
RealtimeToolDispatcher,
RealtimeToolSession,
)
DEFAULT_QWEN_AUDIO_REALTIME_MODEL = "qwen-audio-3.0-realtime-flash"
DEFAULT_QWEN_AUDIO_REALTIME_VOICE = "longanqian"
@@ -43,6 +49,7 @@ QWEN_OUTPUT_SAMPLE_RATE = 24_000
SUPPORTED_TURN_DETECTION_MODES = frozenset({"server_vad", "smart_turn"})
ExtraEventHandler = Callable[[dict[str, Any]], Awaitable[None] | None]
SpeechStartedHandler = Callable[[], Awaitable[None]]
class QwenAudioRealtimeService(AIService):
@@ -115,6 +122,12 @@ class QwenAudioRealtimeService(AIService):
self._user_transcript_item_id = ""
self._user_transcript_timestamp = ""
self._deferred_assistant_messages: list[dict[str, Any]] = []
self._tools: list[RealtimeTool] = []
self._tool_session = RealtimeToolSession(self._send_tool_event)
self._fixed_speech_completion: asyncio.Future[None] | None = None
self._suppress_response_transcript = False
self._speech_started_handler: SpeechStartedHandler | None = None
self._function_names: dict[str, str] = {}
async def start(self, frame: StartFrame) -> None:
await super().start(frame)
@@ -197,6 +210,15 @@ class QwenAudioRealtimeService(AIService):
await self._cancel_active_response()
await self.broadcast_interruption()
async def request_response(self) -> None:
await self._send_event({"type": "response.create"})
def set_speech_started_handler(
self,
handler: SpeechStartedHandler | None,
) -> None:
self._speech_started_handler = handler
async def speak(self, text: str) -> None:
"""Ask Qwen to speak a fixed greeting, then remove the hidden request.
@@ -204,8 +226,21 @@ class QwenAudioRealtimeService(AIService):
instruction field. A temporary user item keeps this behavior within
the supported protocol; it is deleted after the response completes.
"""
await self.speak_fixed(text, suppress_transcript=False)
async def speak_fixed(
self,
text: str,
*,
suppress_transcript: bool = True,
) -> Awaitable[None] | None:
"""Speak configured text and expose the provider response boundary."""
if not text:
return
return None
completion = asyncio.get_running_loop().create_future()
self._resolve_fixed_speech()
self._fixed_speech_completion = completion
self._suppress_response_transcript = suppress_transcript
item_id = f"item_{uuid4().hex}"
self._greeting_request_item_id = item_id
await self._send_event(
@@ -228,6 +263,7 @@ class QwenAudioRealtimeService(AIService):
}
)
await self._send_event({"type": "response.create"})
return completion
async def update_instructions(self, instructions: str) -> None:
"""Update only instructions after startup.
@@ -246,6 +282,33 @@ class QwenAudioRealtimeService(AIService):
wait_until_ready=False,
)
async def update_session(
self,
instructions: str,
tools: list[RealtimeTool],
) -> None:
"""Atomically replace the active Workflow prompt and tool catalog."""
self._instructions = instructions
self._tools = list(tools)
if self._session_ready.is_set():
await self._send_event(
{
"type": "session.update",
"session": {
"instructions": instructions,
"tools": [tool.provider_schema() for tool in tools],
"tool_choice": "auto",
},
},
wait_until_ready=False,
)
def set_tool_dispatcher(
self,
dispatcher: RealtimeToolDispatcher | None,
) -> None:
self._tool_session.set_dispatcher(dispatcher)
def _connection_url(self) -> str:
parts = urlsplit(self._base_url)
query = dict(parse_qsl(parts.query))
@@ -263,6 +326,8 @@ class QwenAudioRealtimeService(AIService):
"output_audio_format": "pcm",
"turn_detection": self._turn_detection_config(),
"max_history_turns": self._max_history_turns,
"tools": [tool.provider_schema() for tool in self._tools],
"tool_choice": "auto",
}
def _turn_detection_config(self) -> dict[str, Any]:
@@ -314,6 +379,9 @@ class QwenAudioRealtimeService(AIService):
self._user_transcript_item_id = ""
self._user_transcript_timestamp = ""
self._deferred_assistant_messages.clear()
self._tool_session.clear()
self._function_names.clear()
self._resolve_fixed_speech()
if websocket and websocket.state is State.OPEN:
try:
await websocket.close()
@@ -368,11 +436,15 @@ class QwenAudioRealtimeService(AIService):
)
)
elif event_type in {"response.audio_transcript.delta", "response.text.delta"}:
if not self._audio_suppressed:
if not self._audio_suppressed and not self._suppress_response_transcript:
await self._append_assistant_text(str(event.get("delta") or ""))
elif event_type in {"response.audio_transcript.done", "response.text.done"}:
transcript = str(event.get("transcript") or event.get("text") or "")
if transcript and not self._audio_suppressed:
if (
transcript
and not self._audio_suppressed
and not self._suppress_response_transcript
):
if self._assistant_turn_id:
self._assistant_text = transcript
else:
@@ -385,6 +457,8 @@ class QwenAudioRealtimeService(AIService):
user_turn_timestamp = time_now_iso8601()
await self._cancel_active_response()
await self.broadcast_interruption()
if self._speech_started_handler is not None:
await self._speech_started_handler()
await self._start_user_transcript_turn(event, user_turn_timestamp)
elif (
event_type == "input_audio_buffer.speech_stopped"
@@ -398,11 +472,20 @@ class QwenAudioRealtimeService(AIService):
self._response_active = False
await self._finish_assistant_text(interrupted=interrupted)
await self._delete_greeting_request()
self._resolve_fixed_speech()
elif event_type == "response.output_item.added":
self._remember_function_call(event)
elif event_type in {
"response.function_call_arguments.done",
"response.output_item.done",
}:
await self._handle_function_call_event(event)
elif event_type == "error":
error = event.get("error")
message = error.get("message") if isinstance(error, dict) else str(error)
if "cancel" not in str(message).lower():
await self.push_error(f"Qwen-Audio Realtime error: {message}")
self._resolve_fixed_speech()
handler = self._extra_event_handlers.get(event_type)
if handler:
@@ -420,6 +503,52 @@ class QwenAudioRealtimeService(AIService):
)
self._response_active = False
await self._finish_assistant_text(interrupted=True)
self._resolve_fixed_speech()
async def _send_tool_event(self, payload: dict[str, Any]) -> None:
await self._send_event(payload, wait_until_ready=False)
async def _handle_function_call_event(self, event: dict[str, Any]) -> None:
item = event.get("item")
source = item if isinstance(item, dict) else event
if isinstance(item, dict) and item.get("type") != "function_call":
return
call_id = str(
source.get("call_id")
or event.get("call_id")
or source.get("id")
or ""
)
name = str(
source.get("name")
or event.get("name")
or self._function_names.get(call_id)
or ""
)
if not name:
return
await self._tool_session.handle_call(
name=name,
call_id=call_id,
arguments=source.get("arguments", event.get("arguments")),
)
self._function_names.pop(call_id, None)
def _remember_function_call(self, event: dict[str, Any]) -> None:
item = event.get("item")
if not isinstance(item, dict) or item.get("type") != "function_call":
return
call_id = str(item.get("call_id") or item.get("id") or "")
name = str(item.get("name") or "")
if call_id and name:
self._function_names[call_id] = name
def _resolve_fixed_speech(self) -> None:
completion = self._fixed_speech_completion
self._fixed_speech_completion = None
self._suppress_response_transcript = False
if completion is not None and not completion.done():
completion.set_result(None)
async def _delete_greeting_request(self) -> None:
item_id = self._greeting_request_item_id

128
backend/services/pipecat/realtime_tools.py Normal file
View File

@@ -0,0 +1,128 @@
"""Provider-neutral function calling for speech-to-speech sessions."""
from __future__ import annotations
import asyncio
import json
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any
from loguru import logger
@dataclass(frozen=True)
class RealtimeTool:
"""Small JSON-schema tool definition understood by both providers."""
name: str
description: str
properties: dict[str, Any] = field(default_factory=dict)
required: tuple[str, ...] = ()
def provider_schema(self) -> dict[str, Any]:
return {
"type": "function",
"name": self.name,
"description": self.description,
"parameters": {
"type": "object",
"properties": self.properties,
"required": list(self.required),
},
}
@dataclass(frozen=True)
class RealtimeToolResult:
"""Tool output plus whether the model should continue immediately."""
output: dict[str, Any]
continue_response: bool = True
after_output: Callable[[], Awaitable[None]] | None = None
RealtimeToolDispatcher = Callable[
[str, dict[str, Any], str], Awaitable[RealtimeToolResult]
]
SendProviderEvent = Callable[[dict[str, Any]], Awaitable[None]]
class RealtimeToolSession:
"""Serialize provider calls and answer every call id at most once."""
def __init__(self, send_event: SendProviderEvent) -> None:
self._send_event = send_event
self._dispatcher: RealtimeToolDispatcher | None = None
self._handled_call_ids: set[str] = set()
self._lock = asyncio.Lock()
def set_dispatcher(self, dispatcher: RealtimeToolDispatcher | None) -> None:
self._dispatcher = dispatcher
async def handle_call(
self,
*,
name: str,
call_id: str,
arguments: str | dict[str, Any] | None,
) -> None:
if not call_id:
logger.warning("Realtime function call 缺少 call_id，已忽略")
return
async with self._lock:
if call_id in self._handled_call_ids:
return
self._handled_call_ids.add(call_id)
parsed = self._parse_arguments(arguments)
try:
if self._dispatcher is None:
result = RealtimeToolResult(
{"status": "error", "message": "当前会话未注册工具处理器"}
)
else:
result = await self._dispatcher(name, parsed, call_id)
except Exception as exc: # noqa: BLE001 - return tool errors to provider
logger.exception(f"Realtime 工具 {name} 执行失败:{exc}")
result = RealtimeToolResult(
{
"status": "error",
"message": f"工具执行失败:{type(exc).__name__}",
}
)
await self._send_event(
{
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": call_id,
"output": json.dumps(
result.output,
ensure_ascii=False,
separators=(",", ":"),
),
},
}
)
if result.after_output is not None:
await result.after_output()
if result.continue_response:
await self._send_event({"type": "response.create"})
def clear(self) -> None:
self._handled_call_ids.clear()
@staticmethod
def _parse_arguments(
arguments: str | dict[str, Any] | None,
) -> dict[str, Any]:
if isinstance(arguments, dict):
return dict(arguments)
if not arguments:
return {}
try:
parsed = json.loads(arguments)
except (TypeError, json.JSONDecodeError):
return {}
return dict(parsed) if isinstance(parsed, dict) else {}

143
backend/services/pipecat/stepfun_realtime.py
View File

@@ -5,6 +5,7 @@ from __future__ import annotations
import asyncio
import base64
import json
from collections.abc import Awaitable, Callable
from typing import Any
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from uuid import uuid4
@@ -28,7 +29,14 @@ from pipecat.utils.time import time_now_iso8601
from websockets.asyncio.client import connect as websocket_connect
from websockets.protocol import State
from services.pipecat.realtime_tools import (
RealtimeTool,
RealtimeToolDispatcher,
RealtimeToolSession,
)
DEFAULT_STEPFUN_REALTIME_URL = "wss://api.stepfun.com/v1/realtime"
SpeechStartedHandler = Callable[[], Awaitable[None]]
class StepFunRealtimeService(AIService):
@@ -68,6 +76,12 @@ class StepFunRealtimeService(AIService):
self._assistant_turn_id: str | None = None
self._assistant_text = ""
self._assistant_timestamp = ""
self._tools: list[RealtimeTool] = []
self._tool_session = RealtimeToolSession(self._send_tool_event)
self._fixed_speech_completion: asyncio.Future[None] | None = None
self._suppress_response_transcript = False
self._speech_started_handler: SpeechStartedHandler | None = None
self._function_names: dict[str, str] = {}
async def start(self, frame: StartFrame) -> None:
await super().start(frame)
@@ -120,6 +134,7 @@ class StepFunRealtimeService(AIService):
if isinstance(frame, InterruptionFrame):
await self._send_event({"type": "response.cancel"}, wait_until_ready=False)
await self._finish_assistant_text(interrupted=True)
self._resolve_fixed_speech()
await self.push_frame(frame, direction)
@@ -140,12 +155,35 @@ class StepFunRealtimeService(AIService):
async def interrupt(self) -> None:
await self._send_event({"type": "response.cancel"}, wait_until_ready=False)
await self._finish_assistant_text(interrupted=True)
self._resolve_fixed_speech()
await self.broadcast_interruption()
async def request_response(self) -> None:
await self._send_event({"type": "response.create"})
def set_speech_started_handler(
self,
handler: SpeechStartedHandler | None,
) -> None:
self._speech_started_handler = handler
async def speak(self, text: str) -> None:
"""Ask the realtime model to voice a fixed greeting."""
await self.speak_fixed(text, suppress_transcript=False)
async def speak_fixed(
self,
text: str,
*,
suppress_transcript: bool = True,
) -> Awaitable[None] | None:
"""Speak configured text and expose the provider response boundary."""
if not text:
return
return None
completion = asyncio.get_running_loop().create_future()
self._resolve_fixed_speech()
self._fixed_speech_completion = completion
self._suppress_response_transcript = suppress_transcript
await self._send_event(
{
"type": "response.create",
@@ -154,6 +192,7 @@ class StepFunRealtimeService(AIService):
},
}
)
return completion
async def _connect(self) -> None:
if self._websocket and self._websocket.state is State.OPEN:
@@ -186,6 +225,9 @@ class StepFunRealtimeService(AIService):
websocket = self._websocket
self._websocket = None
self._session_ready.clear()
self._tool_session.clear()
self._function_names.clear()
self._resolve_fixed_speech()
if websocket and websocket.state is State.OPEN:
try:
await websocket.close()
@@ -240,10 +282,11 @@ class StepFunRealtimeService(AIService):
)
)
elif event_type in {"response.audio_transcript.delta", "response.text.delta"}:
await self._append_assistant_text(str(event.get("delta") or ""))
if not self._suppress_response_transcript:
await self._append_assistant_text(str(event.get("delta") or ""))
elif event_type in {"response.audio_transcript.done", "response.text.done"}:
transcript = str(event.get("transcript") or event.get("text") or "")
if transcript:
if transcript and not self._suppress_response_transcript:
if not self._assistant_turn_id:
await self._append_assistant_text(transcript)
else:
@@ -254,6 +297,9 @@ class StepFunRealtimeService(AIService):
elif event_type == "input_audio_buffer.speech_started":
await self._send_event({"type": "response.cancel"}, wait_until_ready=False)
await self.broadcast_interruption()
self._resolve_fixed_speech()
if self._speech_started_handler is not None:
await self._speech_started_handler()
elif event_type == "response.done":
response = event.get("response")
interrupted = isinstance(response, dict) and response.get("status") in {
@@ -262,11 +308,20 @@ class StepFunRealtimeService(AIService):
"interrupted",
}
await self._finish_assistant_text(interrupted=interrupted)
self._resolve_fixed_speech()
elif event_type == "response.output_item.added":
self._remember_function_call(event)
elif event_type in {
"response.function_call_arguments.done",
"response.output_item.done",
}:
await self._handle_function_call_event(event)
elif event_type == "error":
error = event.get("error")
message = error.get("message") if isinstance(error, dict) else str(error)
if "cancel" not in str(message).lower():
await self.push_error(f"StepFun Realtime error: {message}")
self._resolve_fixed_speech()
async def _send_session_update(self) -> None:
await self._send_event(
@@ -284,6 +339,8 @@ class StepFunRealtimeService(AIService):
"silence_duration_ms": self._silence_duration_ms,
"energy_awakeness_threshold": self._energy_awakeness_threshold,
},
"tools": [tool.provider_schema() for tool in self._tools],
"tool_choice": "auto",
},
},
wait_until_ready=False,
@@ -293,7 +350,85 @@ class StepFunRealtimeService(AIService):
"""Refresh model instructions without rebuilding the realtime session."""
self._instructions = instructions
if self._session_ready.is_set():
await self._send_session_update()
await self._send_event(
{
"type": "session.update",
"session": {"instructions": instructions},
},
wait_until_ready=False,
)
async def update_session(
self,
instructions: str,
tools: list[RealtimeTool],
) -> None:
"""Atomically replace the active Workflow prompt and tool catalog."""
self._instructions = instructions
self._tools = list(tools)
if self._session_ready.is_set():
await self._send_event(
{
"type": "session.update",
"session": {
"instructions": instructions,
"tools": [tool.provider_schema() for tool in tools],
"tool_choice": "auto",
},
},
wait_until_ready=False,
)
def set_tool_dispatcher(
self,
dispatcher: RealtimeToolDispatcher | None,
) -> None:
self._tool_session.set_dispatcher(dispatcher)
async def _send_tool_event(self, payload: dict[str, Any]) -> None:
await self._send_event(payload, wait_until_ready=False)
async def _handle_function_call_event(self, event: dict[str, Any]) -> None:
item = event.get("item")
source = item if isinstance(item, dict) else event
if isinstance(item, dict) and item.get("type") != "function_call":
return
call_id = str(
source.get("call_id")
or event.get("call_id")
or source.get("id")
or ""
)
name = str(
source.get("name")
or event.get("name")
or self._function_names.get(call_id)
or ""
)
if not name:
return
await self._tool_session.handle_call(
name=name,
call_id=call_id,
arguments=source.get("arguments", event.get("arguments")),
)
self._function_names.pop(call_id, None)
def _remember_function_call(self, event: dict[str, Any]) -> None:
item = event.get("item")
if not isinstance(item, dict) or item.get("type") != "function_call":
return
call_id = str(item.get("call_id") or item.get("id") or "")
name = str(item.get("name") or "")
if call_id and name:
self._function_names[call_id] = name
def _resolve_fixed_speech(self) -> None:
completion = self._fixed_speech_completion
self._fixed_speech_completion = None
self._suppress_response_transcript = False
if completion is not None and not completion.done():
completion.set_result(None)
async def _send_event(
self, payload: dict[str, Any], *, wait_until_ready: bool = True