feat: add configurable client tools and photo input

backend/services/pipecat/processors.py

@@ -2,6 +2,7 @@
 
 import asyncio
 from collections.abc import Callable
+from dataclasses import dataclass
 from uuid import uuid4
 
 from loguru import logger
@@ -12,6 +13,11 @@ from services.knowledge import search as search_knowledge
 from db.session import SessionLocal
 
 from pipecat.frames.frames import (
+    BotStartedSpeakingFrame,
+    BotStoppedSpeakingFrame,
+    FunctionCallCancelFrame,
+    FunctionCallResultFrame,
+    FunctionCallsStartedFrame,
     InputTransportMessageFrame,
     InterruptionFrame,
     LLMContextFrame,
@@ -34,42 +40,120 @@ from pipecat.utils.time import time_now_iso8601
 KNOWLEDGE_CONTEXT_MARKER = "<!-- knowledge-context -->"
 
 
-def _text_input(message) -> tuple[str, bool] | None:
-    """解析现有 user-text 与 RTVI send-text 两种前端文字消息。"""
+@dataclass(frozen=True)
+class UserInput:
+    """Validated application input submitted as one user turn."""
+
+    input_id: str
+    text: str
+    has_camera_frame: bool
+    run_immediately: bool
+    interrupt: bool
+
+    @property
+    def prompt_text(self) -> str:
+        if self.text:
+            return self.text
+        return "请根据用户刚提交的图片进行回复。"
+
+    @property
+    def transcript_text(self) -> str:
+        if not self.has_camera_frame:
+            return self.text
+        return f"{self.text}\n已发送一张图片".strip()
+
+
+class UserInputError(ValueError):
+    def __init__(self, message: str, *, input_id: str = "") -> None:
+        super().__init__(message)
+        self.input_id = input_id
+
+
+def parse_user_input(message) -> UserInput | None:
+    """Parse the sole public user-input wire format."""
     if not isinstance(message, dict):
         return None
-    if message.get("type") == "user-text":
-        text = str(message.get("text") or "").strip()
-        return (text, True) if text else None
-    if message.get("type") == "send-text":
-        data = message.get("data")
-        if not isinstance(data, dict):
-            return None
-        text = str(data.get("content") or "").strip()
-        options = data.get("options")
-        run_immediately = not isinstance(options, dict) or options.get(
-            "run_immediately", True
-        )
-        return (text, bool(run_immediately)) if text else None
-    return None
+    if message.get("type") != "user-input":
+        return None
+
+    input_id = str(message.get("input_id") or "").strip()
+    if not input_id or len(input_id) > 128:
+        raise UserInputError("user-input 缺少有效的 input_id", input_id=input_id)
+    if message.get("schema_version") != 1:
+        raise UserInputError("不支持的 user-input schema_version", input_id=input_id)
+
+    parts = message.get("parts")
+    if not isinstance(parts, list) or not parts:
+        raise UserInputError("user-input parts 不能为空", input_id=input_id)
+
+    text = ""
+    has_camera_frame = False
+    for part in parts:
+        if not isinstance(part, dict):
+            raise UserInputError("user-input part 格式不正确", input_id=input_id)
+        part_type = part.get("type")
+        if part_type == "input_text":
+            if text:
+                raise UserInputError("P0 只支持一个 input_text", input_id=input_id)
+            text = str(part.get("text") or "").strip()
+            if not text:
+                raise UserInputError("input_text 不能为空", input_id=input_id)
+        elif part_type == "input_image":
+            if has_camera_frame:
+                raise UserInputError("P0 只支持一张图片", input_id=input_id)
+            source = part.get("source")
+            if (
+                not isinstance(source, dict)
+                or source.get("type") != "camera_frame"
+                or source.get("frame") != "current"
+            ):
+                raise UserInputError(
+                    "P0 只支持当前摄像头画面",
+                    input_id=input_id,
+                )
+            has_camera_frame = True
+        else:
+            raise UserInputError(f"不支持的输入类型: {part_type}", input_id=input_id)
+
+    if not text and not has_camera_frame:
+        raise UserInputError("user-input 没有有效内容", input_id=input_id)
+
+    options = message.get("options")
+    options = options if isinstance(options, dict) else {}
+    run_immediately = bool(options.get("run_immediately", True))
+    interrupt = bool(options.get("interrupt", True))
+    return UserInput(
+        input_id=input_id,
+        text=text,
+        has_camera_frame=has_camera_frame,
+        run_immediately=run_immediately,
+        interrupt=interrupt,
+    )
 
 
-class TextInputProcessor(FrameProcessor):
-    """把 transport 文字消息转换成 LLM 可消费的帧。
+def message_text(message: dict) -> str:
+    """Extract textual content from plain or multimodal LLM messages."""
+    content = message.get("content")
+    if isinstance(content, str):
+        return content.strip()
+    if not isinstance(content, list):
+        return ""
+    return "\n".join(
+        str(part.get("text") or "").strip()
+        for part in content
+        if isinstance(part, dict)
+        and part.get("type") in {"text", "input_text"}
+        and str(part.get("text") or "").strip()
+    ).strip()
 
-    run_immediately(默认/打断):先通过 on_text_input 事件把用户文字交给
-    run_pipeline 登记,再用 broadcast_interruption() 打断当前播报。新的 LLM
-    回复由 assistant aggregator 确认处理完 interruption 后触发。
-    run_immediately=False(RTVI send-text 静默追加):仅把文字写进上下文,
-    不打断、不触发推理。
-    """
+
+class UserInputProcessor(FrameProcessor):
+    """Validate app inputs and coordinate their interruption boundary."""
 
     def __init__(self, should_ignore_input: Callable[[], bool] | None = None):
         super().__init__()
         self._should_ignore_input = should_ignore_input or (lambda: False)
-        # 立即触发的文字(含打断语义)走 on_text_input;静默追加另走一条事件
-        self._register_event_handler("on_text_input")
-        self._register_event_handler("on_text_append")
+        self._register_event_handler("on_user_input")
         self._register_event_handler("on_client_ready")
 
     async def process_frame(self, frame, direction: FrameDirection):
@@ -83,23 +167,35 @@ class TextInputProcessor(FrameProcessor):
             await self._call_event_handler("on_client_ready")
             return
 
-        parsed = _text_input(frame.message)
-        if not parsed:
+        try:
+            user_input = parse_user_input(frame.message)
+        except UserInputError as exc:
+            await self._emit_result(exc.input_id, "error", str(exc))
+            return
+        if user_input is None:
             await self.push_frame(frame, direction)
             return
 
         if self._should_ignore_input():
             logger.debug("通话正在结束,忽略后续文字输入")
+            await self._emit_result(user_input.input_id, "error", "当前不能接收新的用户输入")
             return
 
-        text, run_immediately = parsed
-        if run_immediately:
-            # 先登记文字再打断。下一轮 LLM 由 assistant aggregator 在真正处理完
-            # InterruptionFrame 后触发,避免新回复被这次 interruption 一起取消。
-            await self._call_event_handler("on_text_input", text)
+        await self._call_event_handler("on_user_input", user_input)
+        if user_input.run_immediately and user_input.interrupt:
             await self.broadcast_interruption()
-        else:
-            await self._call_event_handler("on_text_append", text)
+
+    async def _emit_result(self, input_id: str, status: str, message: str = "") -> None:
+        await self.push_frame(
+            OutputTransportMessageUrgentFrame(
+                message={
+                    "type": "user-input-result",
+                    "input_id": input_id,
+                    "status": status,
+                    **({"message": message} if message else {}),
+                }
+            )
+        )
 
 
 class CallEndingUserMuteStrategy(BaseUserMuteStrategy):
@@ -114,6 +210,84 @@ class CallEndingUserMuteStrategy(BaseUserMuteStrategy):
         return self._is_call_ending()
 
 
+@dataclass
+class _BlockedToolCall:
+    tool_finished: bool = False
+    response_started: bool = False
+    response_finished: bool = False
+
+
+class ToolInterruptionUserMuteStrategy(BaseUserMuteStrategy):
+    """Mute user media for selected tools and their associated bot response.
+
+    Pipecat's ``cancel_on_interruption`` also controls whether a tool is
+    synchronous or asynchronous.  This strategy keeps the independent product
+    setting ("allow user interruptions") at the user-input boundary.
+    """
+
+    def __init__(
+        self,
+        blocked_tools: dict[str, str] | set[str],
+    ):
+        super().__init__()
+        self._blocked_tools = (
+            dict(blocked_tools)
+            if isinstance(blocked_tools, dict)
+            else {name: "immediate" for name in blocked_tools}
+        )
+        self._calls: dict[str, _BlockedToolCall] = {}
+        self._bot_speaking = False
+
+    @property
+    def is_muted(self) -> bool:
+        return bool(self._calls)
+
+    async def process_frame(self, frame) -> bool:
+        await super().process_frame(frame)
+
+        if isinstance(frame, BotStartedSpeakingFrame):
+            self._bot_speaking = True
+            for call in self._calls.values():
+                call.response_started = True
+        elif isinstance(frame, BotStoppedSpeakingFrame):
+            self._bot_speaking = False
+            for call in self._calls.values():
+                if call.response_started:
+                    call.response_finished = True
+            self._release_completed()
+        elif isinstance(frame, FunctionCallsStartedFrame):
+            for call in frame.function_calls:
+                execution_mode = self._blocked_tools.get(call.function_name)
+                if execution_mode:
+                    self._calls[call.tool_call_id] = _BlockedToolCall(
+                        # An async tool may continue within the current model
+                        # turn. An immediate tool's current speech is a preamble;
+                        # its protected follow-up starts only after the result.
+                        response_started=(
+                            self._bot_speaking and execution_mode == "async"
+                        ),
+                    )
+        elif isinstance(frame, FunctionCallResultFrame):
+            call = self._calls.get(frame.tool_call_id)
+            if call is not None:
+                call.tool_finished = True
+                self._release_completed()
+        elif isinstance(frame, FunctionCallCancelFrame):
+            # A canceled call cannot reliably produce a follow-up response.
+            self._calls.pop(frame.tool_call_id, None)
+
+        return self.is_muted
+
+    def _release_completed(self) -> None:
+        completed = [
+            call_id
+            for call_id, state in self._calls.items()
+            if state.tool_finished and state.response_finished
+        ]
+        for call_id in completed:
+            self._calls.pop(call_id, None)
+
+
 class VisionCaptureProcessor(FrameProcessor):
     """Capture one requested video frame for auxiliary vision-model analysis."""
 
@@ -193,13 +367,12 @@ class RealtimeDynamicVariableProcessor(FrameProcessor):
         await self.push_frame(frame, direction)
 
 
-class RealtimeTextInputProcessor(FrameProcessor):
-    """Route text input directly to a realtime service without cascade semantics."""
+class RealtimeUserInputProcessor(FrameProcessor):
+    """Route text-only user-input messages to a realtime service."""
 
     def __init__(self):
         super().__init__()
-        self._register_event_handler("on_text_input")
-        self._register_event_handler("on_text_append")
+        self._register_event_handler("on_user_input")
 
     async def process_frame(self, frame, direction: FrameDirection):
         await super().process_frame(frame, direction)
@@ -208,15 +381,32 @@ class RealtimeTextInputProcessor(FrameProcessor):
             await self.push_frame(frame, direction)
             return
 
-        parsed = _text_input(frame.message)
-        if not parsed:
+        try:
+            user_input = parse_user_input(frame.message)
+        except UserInputError as exc:
+            await self._emit_error(exc.input_id, str(exc))
+            return
+        if user_input is None:
             await self.push_frame(frame, direction)
             return
+        if user_input.has_camera_frame:
+            await self._emit_error(
+                user_input.input_id,
+                "Realtime 模式暂不支持图片输入",
+            )
+            return
+        await self._call_event_handler("on_user_input", user_input)
 
-        text, run_immediately = parsed
-        await self._call_event_handler(
-            "on_text_input" if run_immediately else "on_text_append",
-            text,
+    async def _emit_error(self, input_id: str, message: str) -> None:
+        await self.push_frame(
+            OutputTransportMessageUrgentFrame(
+                message={
+                    "type": "user-input-result",
+                    "input_id": input_id,
+                    "status": "error",
+                    "message": message,
+                }
+            )
         )
 
 
@@ -304,7 +494,7 @@ class KnowledgeRetrievalProcessor(FrameProcessor):
         if not user_messages:
             await self.push_frame(frame, direction)
             return
-        query = str(user_messages[-1].get("content") or "").strip()
+        query = message_text(user_messages[-1])
         signature = f"{len(user_messages)}:{query}"
         if not query or signature == self._last_signature:
             await self.push_frame(frame, direction)
@@ -354,8 +544,7 @@ class UserTurnRoutingProcessor(FrameProcessor):
                 message
                 for message in reversed(frame.context.get_messages())
                 if message.get("role") == "user"
-                and isinstance(message.get("content"), str)
-                and str(message.get("content") or "").strip()
+                and message_text(message)
             ),
             None,
         )
@@ -370,7 +559,7 @@ class UserTurnRoutingProcessor(FrameProcessor):
             return
         self._last_user_message = user_message
 
-        content = str(user_message.get("content") or "").strip()
+        content = message_text(user_message)
         handled = await self._brain.on_user_turn_end(content)
         if not handled:
             await self.push_frame(frame, direction)
@@ -447,6 +636,3 @@ class WorkflowAggregatorPair:
 
     def assistant(self):
         return self._assistant
-
-
-