feat(workflow): add edge-tool routing and realtime runtime

backend/services/pipecat/qwen_audio_realtime.py

@@ -35,6 +35,12 @@ from pipecat.utils.time import time_now_iso8601
 from websockets.asyncio.client import connect as websocket_connect
 from websockets.protocol import State
 
+from services.pipecat.realtime_tools import (
+    RealtimeTool,
+    RealtimeToolDispatcher,
+    RealtimeToolSession,
+)
+
 
 DEFAULT_QWEN_AUDIO_REALTIME_MODEL = "qwen-audio-3.0-realtime-flash"
 DEFAULT_QWEN_AUDIO_REALTIME_VOICE = "longanqian"
@@ -43,6 +49,7 @@ QWEN_OUTPUT_SAMPLE_RATE = 24_000
 SUPPORTED_TURN_DETECTION_MODES = frozenset({"server_vad", "smart_turn"})
 
 ExtraEventHandler = Callable[[dict[str, Any]], Awaitable[None] | None]
+SpeechStartedHandler = Callable[[], Awaitable[None]]
 
 
 class QwenAudioRealtimeService(AIService):
@@ -115,6 +122,12 @@ class QwenAudioRealtimeService(AIService):
         self._user_transcript_item_id = ""
         self._user_transcript_timestamp = ""
         self._deferred_assistant_messages: list[dict[str, Any]] = []
+        self._tools: list[RealtimeTool] = []
+        self._tool_session = RealtimeToolSession(self._send_tool_event)
+        self._fixed_speech_completion: asyncio.Future[None] | None = None
+        self._suppress_response_transcript = False
+        self._speech_started_handler: SpeechStartedHandler | None = None
+        self._function_names: dict[str, str] = {}
 
     async def start(self, frame: StartFrame) -> None:
         await super().start(frame)
@@ -197,6 +210,15 @@ class QwenAudioRealtimeService(AIService):
         await self._cancel_active_response()
         await self.broadcast_interruption()
 
+    async def request_response(self) -> None:
+        await self._send_event({"type": "response.create"})
+
+    def set_speech_started_handler(
+        self,
+        handler: SpeechStartedHandler | None,
+    ) -> None:
+        self._speech_started_handler = handler
+
     async def speak(self, text: str) -> None:
         """Ask Qwen to speak a fixed greeting, then remove the hidden request.
 
@@ -204,8 +226,21 @@ class QwenAudioRealtimeService(AIService):
         instruction field. A temporary user item keeps this behavior within
         the supported protocol; it is deleted after the response completes.
         """
+        await self.speak_fixed(text, suppress_transcript=False)
+
+    async def speak_fixed(
+        self,
+        text: str,
+        *,
+        suppress_transcript: bool = True,
+    ) -> Awaitable[None] | None:
+        """Speak configured text and expose the provider response boundary."""
         if not text:
-            return
+            return None
+        completion = asyncio.get_running_loop().create_future()
+        self._resolve_fixed_speech()
+        self._fixed_speech_completion = completion
+        self._suppress_response_transcript = suppress_transcript
         item_id = f"item_{uuid4().hex}"
         self._greeting_request_item_id = item_id
         await self._send_event(
@@ -228,6 +263,7 @@ class QwenAudioRealtimeService(AIService):
             }
         )
         await self._send_event({"type": "response.create"})
+        return completion
 
     async def update_instructions(self, instructions: str) -> None:
         """Update only instructions after startup.
@@ -246,6 +282,33 @@ class QwenAudioRealtimeService(AIService):
                 wait_until_ready=False,
             )
 
+    async def update_session(
+        self,
+        instructions: str,
+        tools: list[RealtimeTool],
+    ) -> None:
+        """Atomically replace the active Workflow prompt and tool catalog."""
+        self._instructions = instructions
+        self._tools = list(tools)
+        if self._session_ready.is_set():
+            await self._send_event(
+                {
+                    "type": "session.update",
+                    "session": {
+                        "instructions": instructions,
+                        "tools": [tool.provider_schema() for tool in tools],
+                        "tool_choice": "auto",
+                    },
+                },
+                wait_until_ready=False,
+            )
+
+    def set_tool_dispatcher(
+        self,
+        dispatcher: RealtimeToolDispatcher | None,
+    ) -> None:
+        self._tool_session.set_dispatcher(dispatcher)
+
     def _connection_url(self) -> str:
         parts = urlsplit(self._base_url)
         query = dict(parse_qsl(parts.query))
@@ -263,6 +326,8 @@ class QwenAudioRealtimeService(AIService):
             "output_audio_format": "pcm",
             "turn_detection": self._turn_detection_config(),
             "max_history_turns": self._max_history_turns,
+            "tools": [tool.provider_schema() for tool in self._tools],
+            "tool_choice": "auto",
         }
 
     def _turn_detection_config(self) -> dict[str, Any]:
@@ -314,6 +379,9 @@ class QwenAudioRealtimeService(AIService):
         self._user_transcript_item_id = ""
         self._user_transcript_timestamp = ""
         self._deferred_assistant_messages.clear()
+        self._tool_session.clear()
+        self._function_names.clear()
+        self._resolve_fixed_speech()
         if websocket and websocket.state is State.OPEN:
             try:
                 await websocket.close()
@@ -368,11 +436,15 @@ class QwenAudioRealtimeService(AIService):
                     )
                 )
         elif event_type in {"response.audio_transcript.delta", "response.text.delta"}:
-            if not self._audio_suppressed:
+            if not self._audio_suppressed and not self._suppress_response_transcript:
                 await self._append_assistant_text(str(event.get("delta") or ""))
         elif event_type in {"response.audio_transcript.done", "response.text.done"}:
             transcript = str(event.get("transcript") or event.get("text") or "")
-            if transcript and not self._audio_suppressed:
+            if (
+                transcript
+                and not self._audio_suppressed
+                and not self._suppress_response_transcript
+            ):
                 if self._assistant_turn_id:
                     self._assistant_text = transcript
                 else:
@@ -385,6 +457,8 @@ class QwenAudioRealtimeService(AIService):
             user_turn_timestamp = time_now_iso8601()
             await self._cancel_active_response()
             await self.broadcast_interruption()
+            if self._speech_started_handler is not None:
+                await self._speech_started_handler()
             await self._start_user_transcript_turn(event, user_turn_timestamp)
         elif (
             event_type == "input_audio_buffer.speech_stopped"
@@ -398,11 +472,20 @@ class QwenAudioRealtimeService(AIService):
             self._response_active = False
             await self._finish_assistant_text(interrupted=interrupted)
             await self._delete_greeting_request()
+            self._resolve_fixed_speech()
+        elif event_type == "response.output_item.added":
+            self._remember_function_call(event)
+        elif event_type in {
+            "response.function_call_arguments.done",
+            "response.output_item.done",
+        }:
+            await self._handle_function_call_event(event)
         elif event_type == "error":
             error = event.get("error")
             message = error.get("message") if isinstance(error, dict) else str(error)
             if "cancel" not in str(message).lower():
                 await self.push_error(f"Qwen-Audio Realtime error: {message}")
+            self._resolve_fixed_speech()
 
         handler = self._extra_event_handlers.get(event_type)
         if handler:
@@ -420,6 +503,52 @@ class QwenAudioRealtimeService(AIService):
             )
         self._response_active = False
         await self._finish_assistant_text(interrupted=True)
+        self._resolve_fixed_speech()
+
+    async def _send_tool_event(self, payload: dict[str, Any]) -> None:
+        await self._send_event(payload, wait_until_ready=False)
+
+    async def _handle_function_call_event(self, event: dict[str, Any]) -> None:
+        item = event.get("item")
+        source = item if isinstance(item, dict) else event
+        if isinstance(item, dict) and item.get("type") != "function_call":
+            return
+        call_id = str(
+            source.get("call_id")
+            or event.get("call_id")
+            or source.get("id")
+            or ""
+        )
+        name = str(
+            source.get("name")
+            or event.get("name")
+            or self._function_names.get(call_id)
+            or ""
+        )
+        if not name:
+            return
+        await self._tool_session.handle_call(
+            name=name,
+            call_id=call_id,
+            arguments=source.get("arguments", event.get("arguments")),
+        )
+        self._function_names.pop(call_id, None)
+
+    def _remember_function_call(self, event: dict[str, Any]) -> None:
+        item = event.get("item")
+        if not isinstance(item, dict) or item.get("type") != "function_call":
+            return
+        call_id = str(item.get("call_id") or item.get("id") or "")
+        name = str(item.get("name") or "")
+        if call_id and name:
+            self._function_names[call_id] = name
+
+    def _resolve_fixed_speech(self) -> None:
+        completion = self._fixed_speech_completion
+        self._fixed_speech_completion = None
+        self._suppress_response_transcript = False
+        if completion is not None and not completion.done():
+            completion.set_result(None)
 
     async def _delete_greeting_request(self) -> None:
         item_id = self._greeting_request_item_id