feat: add configurable client tools and photo input

backend/services/pipecat/pipeline_events.py

@@ -1,12 +1,13 @@
 """Event registration for cascade and realtime conversation pipelines."""
 
+from collections.abc import Awaitable, Callable
+
 from loguru import logger
 
 from pipecat.frames.frames import (
     BotStartedSpeakingFrame,
     BotStoppedSpeakingFrame,
     EndFrame,
-    LLMMessagesAppendFrame,
     OutputTransportMessageUrgentFrame,
     TTSSpeakFrame,
 )
@@ -15,6 +16,7 @@ from pipecat.runner.utils import (
     maybe_capture_participant_camera,
 )
 from pipecat.utils.time import time_now_iso8601
+from services.pipecat.processors import UserInput
 
 
 def bind_cascade_pipeline_events(
@@ -29,10 +31,11 @@ def bind_cascade_pipeline_events(
     greeting: str,
     vision_enabled: bool,
     vision_state: dict[str, str | None],
+    submit_user_input: Callable[[UserInput], Awaitable[None]] | None = None,
 ) -> None:
     """Connect processors to transport events without owning pipeline assembly."""
 
-    pending_text_inputs: list[str] = []
+    pending_user_inputs: list[UserInput] = []
     greeting_transcript_sent = False
     greeting_timestamp = ""
     greeting_playback_pending = False
@@ -72,14 +75,33 @@ def bind_cascade_pipeline_events(
             )
         )
 
-    async def append_user_text_to_context(text: str, *, run_llm: bool) -> None:
+    async def queue_input_result(
+        user_input: UserInput,
+        status: str,
+        message: str = "",
+    ) -> None:
         await worker.queue_frame(
-            LLMMessagesAppendFrame(
-                messages=[{"role": "user", "content": text}],
-                run_llm=run_llm,
+            OutputTransportMessageUrgentFrame(
+                message={
+                    "type": "user-input-result",
+                    "input_id": user_input.input_id,
+                    "status": status,
+                    **({"message": message} if message else {}),
+                }
             )
         )
 
+    async def finish_user_input(user_input: UserInput) -> None:
+        try:
+            if submit_user_input is None:
+                raise RuntimeError("用户输入提交器尚未配置")
+            await submit_user_input(user_input)
+        except Exception as exc:  # noqa: BLE001 - input errors must reach the client
+            logger.warning(f"用户输入处理失败: {exc}")
+            await queue_input_result(user_input, "error", str(exc))
+            return
+        await queue_input_result(user_input, "accepted")
+
     @user_aggregator.event_handler("on_user_turn_stopped")
     async def on_user_turn_stopped(_aggregator, _strategy, message):
         await queue_transcript("user", message.content, message.timestamp)
@@ -123,24 +145,23 @@ def bind_cascade_pipeline_events(
         )
         await brain.on_assistant_text_end(turn_id, content, interrupted)
 
-    @text_input.event_handler("on_text_input")
-    async def on_text_input(_processor, text):
-        pending_text_inputs.append(text)
-        # The transcript must be queued before the interruption is broadcast.
-        await queue_transcript("user", text, time_now_iso8601())
+    @text_input.event_handler("on_user_input")
+    async def on_user_input(_processor, user_input: UserInput):
+        await queue_transcript(
+            "user",
+            user_input.transcript_text,
+            time_now_iso8601(),
+        )
+        if user_input.run_immediately and user_input.interrupt:
+            pending_user_inputs.append(user_input)
+            return
+        await finish_user_input(user_input)
 
     @assistant_aggregator.event_handler("on_interruption_processed")
     async def on_interruption_processed(_aggregator):
-        if not pending_text_inputs:
+        if not pending_user_inputs:
             return
-        text = pending_text_inputs.pop(0)
-        await append_user_text_to_context(text, run_llm=True)
-
-    @text_input.event_handler("on_text_append")
-    async def on_text_append(_processor, text):
-        brain.record_user_message(text)
-        await queue_transcript("user", text, time_now_iso8601())
-        await append_user_text_to_context(text, run_llm=False)
+        await finish_user_input(pending_user_inputs.pop(0))
 
     @text_input.event_handler("on_client_ready")
     async def on_client_ready(_processor):
@@ -218,16 +239,24 @@ def bind_realtime_pipeline_events(
             )
         )
 
-    @text_input.event_handler("on_text_input")
-    async def on_text_input(_processor, text):
-        await queue_transcript("user", text)
-        await realtime.interrupt()
-        await realtime.send_text(text, run_immediately=True)
-
-    @text_input.event_handler("on_text_append")
-    async def on_text_append(_processor, text):
-        await queue_transcript("user", text)
-        await realtime.send_text(text, run_immediately=False)
+    @text_input.event_handler("on_user_input")
+    async def on_user_input(_processor, user_input: UserInput):
+        await queue_transcript("user", user_input.text)
+        if user_input.run_immediately and user_input.interrupt:
+            await realtime.interrupt()
+        await realtime.send_text(
+            user_input.text,
+            run_immediately=user_input.run_immediately,
+        )
+        await worker.queue_frame(
+            OutputTransportMessageUrgentFrame(
+                message={
+                    "type": "user-input-result",
+                    "input_id": user_input.input_id,
+                    "status": "accepted",
+                }
+            )
+        )
 
     @transport.event_handler("on_client_connected")
     async def on_client_connected(_transport, _client):