feat: add configurable client tools and photo input

backend/services/pipecat/pipeline.py

@@ -30,6 +30,8 @@ from services.pipecat.service_factory import (
 )
 from db.session import SessionLocal
 from services.knowledge import search as search_knowledge
+from services.client_tools import ClientToolBroker
+from services.tool_policy import policy_for_tool
 from services.vision import (
     VISION_ANALYSIS_SYSTEM_PROMPT,
     VISION_SYSTEM_HINT,
@@ -44,6 +46,7 @@ from pipecat.adapters.schemas.tools_schema import ToolsSchema
 from pipecat.flows import FlowsFunctionSchema
 from pipecat.frames.frames import (
     EndFrame,
+    LLMMessagesAppendFrame,
     OutputTransportMessageUrgentFrame,
     UserImageRawFrame,
     UserImageRequestFrame,
@@ -57,9 +60,6 @@ from pipecat.processors.aggregators.llm_response_universal import (
 )
 from pipecat.processors.frame_processor import FrameDirection, FrameProcessor
 from pipecat.services.llm_service import FunctionCallParams
-from pipecat.turns.user_mute.function_call_user_mute_strategy import (
-    FunctionCallUserMuteStrategy,
-)
 from services.pipecat.turn_config import (
     ConfigurableLLMUserAggregator,
     create_user_turn_strategies,
@@ -73,11 +73,13 @@ from services.pipecat.processors import (
     KnowledgeRetrievalProcessor,
     PassthroughLLMAssistantAggregator,
     RealtimeDynamicVariableProcessor,
-    RealtimeTextInputProcessor,
-    TextInputProcessor,
+    RealtimeUserInputProcessor,
+    UserInput,
+    UserInputProcessor,
     UserTurnRoutingProcessor,
     VisionCaptureProcessor,
     WorkflowAggregatorPair,
+    ToolInterruptionUserMuteStrategy,
 )
 from services.pipecat.workflow_services import (
     WorkflowServiceController,
@@ -319,6 +321,14 @@ async def run_pipeline(
         )
     )
     input_state = {"enabled": True}
+    tool_interruption_strategy = ToolInterruptionUserMuteStrategy(
+        {
+            tool.function_name: policy_for_tool(tool).execution_mode
+            for tool in cfg.tools
+            if tool.type in {"http", "mcp", "client"}
+            and not policy_for_tool(tool).allow_interruptions
+        }
+    )
     # LLM 槽由大脑提供:本地模型或 Dify/FastGPT 外部托管适配器。
     llm = brain.build_llm(
         config_with_resource(cfg, default_llm_resource)
@@ -335,10 +345,10 @@ async def run_pipeline(
         params=LLMUserAggregatorParams(
             vad_analyzer=create_vad_analyzer(cfg.turnConfig),
             user_mute_strategies=[
-                FunctionCallUserMuteStrategy(),
                 CallEndingUserMuteStrategy(
                     lambda: call_end.ending or not input_state["enabled"]
                 ),
+                tool_interruption_strategy,
             ],
             user_turn_strategies=create_user_turn_strategies(
                 cfg.turnConfig,
@@ -348,9 +358,14 @@ async def run_pipeline(
     )
     user_turn_router = UserTurnRoutingProcessor(brain)
     assistant_aggregator = PassthroughLLMAssistantAggregator(context)
-    text_input = TextInputProcessor(
-        should_ignore_input=lambda: call_end.ending or not input_state["enabled"]
+    user_input = UserInputProcessor(
+        should_ignore_input=lambda: (
+            call_end.ending
+            or not input_state["enabled"]
+            or tool_interruption_strategy.is_muted
+        )
     )
+    client_tools = ClientToolBroker()
     vision_capture = VisionCaptureProcessor()
     knowledge_retrieval = KnowledgeRetrievalProcessor(
         automatic_knowledge_id,
@@ -471,6 +486,24 @@ async def run_pipeline(
 
     flow_global_functions = []
     workflow_vision_function = None
+
+    def active_vision_config() -> AssistantConfig:
+        if cfg.type != "workflow":
+            return cfg
+        if not workflow_vision_scope.get("enabled"):
+            raise ValueError("当前 Workflow Agent 节点未启用视觉能力")
+        vision_resource_id = str(
+            workflow_vision_scope.get("vision_model_resource_id") or ""
+        )
+        llm_resource_id = str(workflow_vision_scope.get("llm_resource_id") or "")
+        resource_id = vision_resource_id or llm_resource_id
+        resource = cfg.workflow_model_resources.get(resource_id)
+        if resource:
+            return config_with_vision_resource(cfg, resource)
+        if not vision_resource_id:
+            return config_with_main_llm_as_vision(cfg)
+        raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
+
     if cfg.type == "workflow" and vision_enabled:
         async def flow_fetch_user_image(args, _flow_manager):
             if not workflow_vision_scope.get("enabled"):
@@ -492,23 +525,9 @@ async def run_pipeline(
                 function_name=VISION_TOOL_NAME,
             )
             try:
-                vision_resource_id = str(
-                    workflow_vision_scope.get("vision_model_resource_id") or ""
-                )
-                llm_resource_id = str(
-                    workflow_vision_scope.get("llm_resource_id") or ""
-                )
-                resource_id = vision_resource_id or llm_resource_id
-                resource = cfg.workflow_model_resources.get(resource_id)
-                if resource:
-                    vision_cfg = config_with_vision_resource(cfg, resource)
-                elif not vision_resource_id:
-                    vision_cfg = config_with_main_llm_as_vision(cfg)
-                else:
-                    raise ValueError(f"视觉模型资源未加载:{vision_resource_id}")
                 frame = await vision_capture.request_image(llm, request)
                 observation = await _analyze_image_with_vision_model(
-                    vision_cfg,
+                    active_vision_config(),
                     frame,
                     question,
                 )
@@ -558,8 +577,9 @@ async def run_pipeline(
     pipeline = Pipeline(
         [
             transport.input(),
+            client_tools,
             vision_capture,
-            text_input,
+            user_input,
             stt_processor,
             user_aggregator,
             user_turn_router,
@@ -638,6 +658,7 @@ async def run_pipeline(
             set_system_prompt=set_system_prompt,
             set_tools=set_visible_tools,
             call_end=call_end,
+            client_tools=client_tools,
             worker=worker,
             context_aggregator=WorkflowAggregatorPair(
                 user_aggregator,
@@ -654,17 +675,78 @@ async def run_pipeline(
         ),
     )
 
+    async def submit_user_input(value: UserInput) -> None:
+        if not value.has_camera_frame:
+            if not value.run_immediately:
+                brain.record_user_message(value.text)
+            await worker.queue_frame(
+                LLMMessagesAppendFrame(
+                    messages=[{"role": "user", "content": value.text}],
+                    run_llm=value.run_immediately,
+                )
+            )
+            return
+
+        if not value.run_immediately:
+            raise ValueError("P0 图片输入必须立即触发回复")
+        if not vision_enabled:
+            raise ValueError("当前助手未启用视觉能力")
+        user_id = vision_state.get("client_id")
+        if not user_id:
+            raise ValueError("当前没有可用的摄像头视频流")
+        analysis_cfg = None if vision_native_mode else active_vision_config()
+
+        request = UserImageRequestFrame(
+            user_id=user_id,
+            text=value.prompt_text,
+            append_to_context=False,
+        )
+        try:
+            image_frame = await vision_capture.request_image(llm, request)
+        except asyncio.TimeoutError as exc:
+            raise ValueError("等待摄像头视频帧超时") from exc
+
+        if vision_native_mode:
+            image_frame.text = value.prompt_text
+            image_frame.append_to_context = True
+            image_frame.request = None
+            await worker.queue_frame(image_frame)
+            return
+
+        try:
+            assert analysis_cfg is not None
+            observation = await _analyze_image_with_vision_model(
+                analysis_cfg,
+                image_frame,
+                value.prompt_text,
+            )
+        except Exception as exc:
+            logger.exception(f"用户图片视觉分析失败: {exc}")
+            raise ValueError("视觉理解暂时不可用") from exc
+        content = (
+            f"{value.prompt_text}\n\n"
+            "[视觉模型对用户刚提交图片的观察]\n"
+            f"{observation or '视觉模型没有返回有效观察结果。'}"
+        )
+        await worker.queue_frame(
+            LLMMessagesAppendFrame(
+                messages=[{"role": "user", "content": content}],
+                run_llm=True,
+            )
+        )
+
     bind_cascade_pipeline_events(
         transport=transport,
         worker=worker,
         brain=brain,
         context=context,
-        text_input=text_input,
+        text_input=user_input,
         user_aggregator=user_aggregator,
         assistant_aggregator=assistant_aggregator,
         greeting=greeting,
         vision_enabled=vision_enabled,
         vision_state=vision_state,
+        submit_user_input=submit_user_input,
     )
     runner = WorkerRunner(handle_sigint=False)
     run_status = "completed"
@@ -694,7 +776,7 @@ async def run_realtime_pipeline(
         instructions=brain.system_prompt(cfg),
     )
     input_sample_rate, output_sample_rate = realtime_audio_sample_rates(cfg)
-    text_input = RealtimeTextInputProcessor()
+    user_input = RealtimeUserInputProcessor()
     dynamic_variables = RealtimeDynamicVariableProcessor(brain, cfg, realtime)
     greeting = await brain.greeting(cfg)
 
@@ -708,7 +790,7 @@ async def run_realtime_pipeline(
     pipeline = Pipeline(
         [
             transport.input(),
-            text_input,
+            user_input,
             realtime,
             dynamic_variables,
             ConversationHistoryProcessor(recorder),
@@ -729,7 +811,7 @@ async def run_realtime_pipeline(
         transport=transport,
         worker=worker,
         realtime=realtime,
-        text_input=text_input,
+        text_input=user_input,
         greeting=greeting,
     )
     runner = WorkerRunner(handle_sigint=False)