fix: relay confirmation interrupts past muted input

This commit is contained in:
Xin Wang
2026-08-04 11:12:34 +08:00
parent da828aca41
commit 775e4058bc
8 changed files with 153 additions and 70 deletions

View File

@@ -95,7 +95,6 @@ class BrainRuntime:
set_vision_scope: Callable[[dict[str, Any]], None] | None = None set_vision_scope: Callable[[dict[str, Any]], None] | None = None
vision_function: Any = None vision_function: Any = None
set_input_enabled: Callable[[bool], None] | None = None set_input_enabled: Callable[[bool], None] | None = None
wait_for_output_stopped: Callable[[], Awaitable[None]] | None = None
apply_turn_config: ( apply_turn_config: (
Callable[[bool, dict[str, Any]], Awaitable[None]] | None Callable[[bool, dict[str, Any]], Awaitable[None]] | None
) = None ) = None

View File

@@ -1093,16 +1093,6 @@ class WorkflowBrain(BaseBrain):
), ),
) )
if result.succeeded: if result.succeeded:
if (
completion_policy == MESSAGE_CONFIRMATION
and result.action == "confirmed"
):
if runtime.wait_for_output_stopped is None:
raise RuntimeError("当前管线不支持等待 Message 确认语音停止")
# ClientToolBroker has already broadcast the interruption
# before resolving the confirmation. Keep the next node behind
# the pipeline and transport playback boundaries.
await runtime.wait_for_output_stopped()
await self._emit_trace( await self._emit_trace(
"message_completed", "message_completed",
nodeId=node_id, nodeId=node_id,

View File

@@ -3,6 +3,8 @@
from __future__ import annotations from __future__ import annotations
import asyncio import asyncio
from collections import deque
from collections.abc import Awaitable, Callable
from dataclasses import dataclass from dataclasses import dataclass
from typing import Any, Literal, Protocol from typing import Any, Literal, Protocol
@@ -30,6 +32,14 @@ class _PendingClientToolCall:
interrupt_on_result: bool = False interrupt_on_result: bool = False
@dataclass(frozen=True)
class _DeferredClientToolResult:
"""A successful result waiting for the media interruption barrier."""
future: asyncio.Future[dict[str, Any]]
response: dict[str, Any]
class ClientToolPort(Protocol): class ClientToolPort(Protocol):
async def call( async def call(
self, self,
@@ -49,8 +59,17 @@ class ClientToolBroker(FrameProcessor):
def __init__(self) -> None: def __init__(self) -> None:
super().__init__() super().__init__()
self._pending: dict[str, _PendingClientToolCall] = {} self._pending: dict[str, _PendingClientToolCall] = {}
self._deferred_results: deque[_DeferredClientToolResult] = deque()
self._interrupt_handler: Callable[[], Awaitable[None]] | None = None
self._closed_message: str | None = None self._closed_message: str | None = None
def set_interrupt_handler(
self,
handler: Callable[[], Awaitable[None]],
) -> None:
"""Use a processor downstream of muted user input as interrupt source."""
self._interrupt_handler = handler
async def call( async def call(
self, self,
function_name: str, function_name: str,
@@ -142,21 +161,42 @@ class ClientToolBroker(FrameProcessor):
logger.debug(f"忽略未知或过期的客户端工具结果: {tool_call_id}") logger.debug(f"忽略未知或过期的客户端工具结果: {tool_call_id}")
return return
status = str(message.get("status") or "error")
# Keep interruption in the input-frame path, matching text input.
# Resolve the tool future only after the interruption has been
# broadcast, so the workflow cannot enqueue its next node first.
if status == "ok" and pending.interrupt_on_result:
await self.broadcast_interruption()
future = pending.future future = pending.future
status = str(message.get("status") or "error")
if status == "ok": if status == "ok":
future.set_result( response = {
{
"status": "ok", "status": "ok",
"data": message.get("data"), "data": message.get("data"),
} }
if pending.interrupt_on_result:
# Match text input exactly: broadcasting only starts the
# interruption. The assistant aggregator calls
# on_interruption_processed() after the old response state has
# actually been cleared; only then may the workflow continue.
deferred = _DeferredClientToolResult(
future=future,
response=response,
) )
self._deferred_results.append(deferred)
try:
if self._interrupt_handler is not None:
await self._interrupt_handler()
else:
await self.broadcast_interruption()
except Exception as exc:
try:
self._deferred_results.remove(deferred)
except ValueError:
# The downstream acknowledgement may have won the
# race before an upstream broadcast failure arrived.
pass
if not future.done():
future.set_exception(
ClientToolError("客户端工具结果中断处理失败")
)
logger.exception(f"客户端工具结果中断失败: {exc}")
return
future.set_result(response)
else: else:
future.set_result( future.set_result(
{ {
@@ -166,7 +206,18 @@ class ClientToolBroker(FrameProcessor):
} }
) )
def on_interruption_processed(self) -> None:
"""Release one result after the aggregator acknowledges interruption."""
while self._deferred_results:
deferred = self._deferred_results.popleft()
if deferred.future.done():
continue
deferred.future.set_result(deferred.response)
logger.debug("客户端工具结果已通过中断处理边界")
return
def _fail_pending(self, message: str) -> None: def _fail_pending(self, message: str) -> None:
self._deferred_results.clear()
for pending in self._pending.values(): for pending in self._pending.values():
future = pending.future future = pending.future
if not future.done(): if not future.done():

View File

@@ -8,7 +8,6 @@
import asyncio import asyncio
import base64 import base64
from collections.abc import Awaitable, Callable
from io import BytesIO from io import BytesIO
from typing import Any from typing import Any
@@ -107,21 +106,6 @@ ON_DEMAND_KNOWLEDGE_SYSTEM_HINT = (
) )
async def _wait_for_output_stop(
wait_until_stopped: Callable[[], Awaitable[None]],
*,
timeout_seconds: float = 3.0,
) -> None:
"""Wait for the transport's real stop event after an interruption."""
try:
await asyncio.wait_for(
wait_until_stopped(),
timeout=timeout_seconds,
)
except TimeoutError as exc:
raise RuntimeError("等待客户端语音停止超时") from exc
def _compact_knowledge_metadata(value: str, max_length: int) -> str: def _compact_knowledge_metadata(value: str, max_length: int) -> str:
"""Keep tool metadata useful without letting it dominate the model context.""" """Keep tool metadata useful without letting it dominate the model context."""
compact = " ".join(value.split()) compact = " ".join(value.split())
@@ -437,6 +421,11 @@ async def run_pipeline(
should_ignore_update=lambda: call_end.ending, should_ignore_update=lambda: call_end.ending,
) )
client_tools = ClientToolBroker() client_tools = ClientToolBroker()
# Confirmation nodes mute the user aggregator, which intentionally drops
# InterruptionFrame. Relay confirmation interruptions from the next
# processor instead so LLM, assistant aggregation, TTS, and output all see
# the frame before the workflow continues.
client_tools.set_interrupt_handler(user_turn_router.broadcast_interruption)
vision_capture = VisionCaptureProcessor() vision_capture = VisionCaptureProcessor()
knowledge_retrieval = KnowledgeRetrievalProcessor( knowledge_retrieval = KnowledgeRetrievalProcessor(
automatic_knowledge_id, automatic_knowledge_id,
@@ -751,11 +740,6 @@ async def run_pipeline(
current_enable_interrupt = enable_interrupt current_enable_interrupt = enable_interrupt
current_turn_config = normalized current_turn_config = normalized
async def wait_for_output_stopped() -> None:
"""Keep workflow continuation behind the interrupted output."""
if call_end.speaking:
await _wait_for_output_stop(call_end.wait_until_silent)
def set_system_prompt(text: str) -> None: def set_system_prompt(text: str) -> None:
"""替换上下文里的系统提示(节点切换时整体替换,而非追加)。""" """替换上下文里的系统提示(节点切换时整体替换,而非追加)。"""
messages = context.get_messages() messages = context.get_messages()
@@ -788,7 +772,6 @@ async def run_pipeline(
set_vision_scope=lambda scope: workflow_vision_scope.update(scope), set_vision_scope=lambda scope: workflow_vision_scope.update(scope),
vision_function=workflow_vision_function, vision_function=workflow_vision_function,
set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled), set_input_enabled=lambda enabled: input_state.__setitem__("enabled", enabled),
wait_for_output_stopped=wait_for_output_stopped,
apply_turn_config=apply_workflow_turn_config, apply_turn_config=apply_workflow_turn_config,
flow_global_functions=flow_global_functions, flow_global_functions=flow_global_functions,
), ),
@@ -877,6 +860,7 @@ async def run_pipeline(
vision_enabled=vision_enabled, vision_enabled=vision_enabled,
vision_state=vision_state, vision_state=vision_state,
submit_user_input=submit_user_input, submit_user_input=submit_user_input,
client_tools=client_tools,
) )
runner = WorkerRunner(handle_sigint=False) runner = WorkerRunner(handle_sigint=False)
run_status = "completed" run_status = "completed"

View File

@@ -16,6 +16,7 @@ from pipecat.runner.utils import (
maybe_capture_participant_camera, maybe_capture_participant_camera,
) )
from pipecat.utils.time import time_now_iso8601 from pipecat.utils.time import time_now_iso8601
from services.client_tools import ClientToolBroker
from services.pipecat.processors import UserInput from services.pipecat.processors import UserInput
@@ -32,6 +33,7 @@ def bind_cascade_pipeline_events(
vision_enabled: bool, vision_enabled: bool,
vision_state: dict[str, str | None], vision_state: dict[str, str | None],
submit_user_input: Callable[[UserInput], Awaitable[None]] | None = None, submit_user_input: Callable[[UserInput], Awaitable[None]] | None = None,
client_tools: ClientToolBroker | None = None,
) -> None: ) -> None:
"""Connect processors to transport events without owning pipeline assembly.""" """Connect processors to transport events without owning pipeline assembly."""
@@ -159,6 +161,8 @@ def bind_cascade_pipeline_events(
@assistant_aggregator.event_handler("on_interruption_processed") @assistant_aggregator.event_handler("on_interruption_processed")
async def on_interruption_processed(_aggregator): async def on_interruption_processed(_aggregator):
if client_tools is not None:
client_tools.on_interruption_processed()
if not pending_user_inputs: if not pending_user_inputs:
return return
await finish_user_input(pending_user_inputs.pop(0)) await finish_user_input(pending_user_inputs.pop(0))

View File

@@ -1541,9 +1541,6 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
): ):
events.append("message_completed") events.append("message_completed")
async def wait_for_output_stopped():
events.append("output_stopped")
class OrderedCallEnd(FakeCallEnd): class OrderedCallEnd(FakeCallEnd):
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -1580,7 +1577,6 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
call_end=call_end, call_end=call_end,
client_tools=client_tools, client_tools=client_tools,
set_input_enabled=input_states.append, set_input_enabled=input_states.append,
wait_for_output_stopped=wait_for_output_stopped,
) )
brain._message_stages.set_client_tools(client_tools) brain._message_stages.set_client_tools(client_tools)
@@ -1606,7 +1602,7 @@ class WorkflowBrainTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(result.action, "confirmed") self.assertEqual(result.action, "confirmed")
self.assertFalse(call_end.playback_completion.done()) self.assertFalse(call_end.playback_completion.done())
self.assertEqual(input_states, [False, True]) self.assertEqual(input_states, [False, True])
self.assertEqual(events[-2:], ["output_stopped", "message_completed"]) self.assertEqual(events[-1], "message_completed")
async def test_speech_only_message_waits_for_transport_playback(self): async def test_speech_only_message_waits_for_transport_playback(self):
brain = WorkflowBrain( brain = WorkflowBrain(

View File

@@ -138,7 +138,52 @@ class ClientToolExecutorTests(unittest.IsolatedAsyncioTestCase):
class ClientToolBrokerTests(unittest.IsolatedAsyncioTestCase): class ClientToolBrokerTests(unittest.IsolatedAsyncioTestCase):
async def test_interrupts_before_resolving_configured_result(self): async def test_uses_configured_downstream_interrupt_handler(self):
broker = ClientToolBroker()
outbound = []
interruptions = []
async def push_frame(frame, direction=FrameDirection.DOWNSTREAM):
outbound.append((frame, direction))
async def interrupt_after_muted_input():
interruptions.append("downstream")
broker.push_frame = push_frame
broker.set_interrupt_handler(interrupt_after_muted_input)
call = asyncio.create_task(
broker.call(
"show_message",
{},
timeout_seconds=1,
response_wait_mode="session",
interrupt_on_result=True,
)
)
await asyncio.sleep(0)
message = outbound[0][0].message
await broker.process_frame(
InputTransportMessageFrame(
message={
"type": "client-tool-result",
"tool_call_id": message["tool_call_id"],
"status": "ok",
"data": {"action": "confirmed"},
}
),
FrameDirection.DOWNSTREAM,
)
self.assertEqual(interruptions, ["downstream"])
self.assertFalse(call.done())
broker.on_interruption_processed()
self.assertEqual(
await call,
{"status": "ok", "data": {"action": "confirmed"}},
)
async def test_resolves_configured_result_after_interruption_processed(self):
broker = ClientToolBroker() broker = ClientToolBroker()
outbound = [] outbound = []
observed_future_states = [] observed_future_states = []
@@ -178,6 +223,10 @@ class ClientToolBrokerTests(unittest.IsolatedAsyncioTestCase):
) )
self.assertEqual(observed_future_states, [[False]]) self.assertEqual(observed_future_states, [[False]])
self.assertFalse(call.done())
broker.on_interruption_processed()
self.assertEqual( self.assertEqual(
await call, await call,
{"status": "ok", "data": {"action": "confirmed"}}, {"status": "ok", "data": {"action": "confirmed"}},

View File

@@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
import asyncio
import unittest import unittest
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import AsyncMock, patch from unittest.mock import AsyncMock, patch
@@ -11,7 +10,6 @@ from pipecat.frames.frames import (
OutputTransportMessageUrgentFrame, OutputTransportMessageUrgentFrame,
) )
from services.pipecat.pipeline_events import bind_cascade_pipeline_events from services.pipecat.pipeline_events import bind_cascade_pipeline_events
from services.pipecat.pipeline import _wait_for_output_stop
class _EventSource: class _EventSource:
@@ -82,26 +80,38 @@ class _Brain:
class PipelineEventTest(unittest.IsolatedAsyncioTestCase): class PipelineEventTest(unittest.IsolatedAsyncioTestCase):
async def test_interrupted_output_waits_for_transport_stop(self): async def test_interruption_acknowledges_deferred_client_tool_result(self):
events = [] transport = _EventSource()
text_input = _EventSource()
async def wait_until_stopped(): user_aggregator = _EventSource()
events.append("stopped") assistant_aggregator = _EventSource()
worker = _Worker()
await _wait_for_output_stop(wait_until_stopped) brain = _Brain(worker)
acknowledgements = []
self.assertEqual(events, ["stopped"]) client_tools = SimpleNamespace(
on_interruption_processed=lambda: acknowledgements.append(True)
async def test_interrupted_output_rejects_stop_timeout(self):
async def wait_forever():
await asyncio.Event().wait()
with self.assertRaisesRegex(RuntimeError, "语音停止超时"):
await _wait_for_output_stop(
wait_forever,
timeout_seconds=0.001,
) )
bind_cascade_pipeline_events(
transport=transport,
worker=worker,
brain=brain,
context=SimpleNamespace(),
text_input=text_input,
user_aggregator=user_aggregator,
assistant_aggregator=assistant_aggregator,
greeting="",
vision_enabled=False,
vision_state={"client_id": None},
client_tools=client_tools,
)
await assistant_aggregator.handlers["on_interruption_processed"](
assistant_aggregator
)
self.assertEqual(acknowledgements, [True])
async def test_greeting_keeps_playback_timestamp_until_client_ready(self): async def test_greeting_keeps_playback_timestamp_until_client_ready(self):
transport = _EventSource() transport = _EventSource()
text_input = _EventSource() text_input = _EventSource()