Implement response handling and speech tracking in CallEnd functionality

- Add begin_response and finish_after_current_speech methods to CallEndCoordinator for better management of speech events.
- Update PromptBrain to utilize new methods, ensuring proper handling of generated closing speech and tool-only calls.
- Enhance tests to verify the correct behavior of speech tracking and response handling in various scenarios, including waiting for audio to finish before ending calls.
- Introduce a new test suite for CallEndCoordinator to validate the interaction with speech frames.

backend/services/brains/base.py

@@ -51,8 +51,12 @@ class CallEndPort(Protocol):
 
     def begin(self, reason: str) -> None: ...
 
+    def begin_response(self) -> None: ...
+
     def arm_after_speech(self) -> None: ...
 
+    async def finish_after_current_speech(self, *, has_text: bool) -> None: ...
+
     async def finish(self) -> None: ...
 
 

backend/services/brains/prompt_brain.py

@@ -35,6 +35,7 @@ class PromptBrain(BaseBrain):
         self._store = DynamicVariableStore.from_config(cfg)
         self._tools = ToolExecutor(self._store)
         self._runtime: BrainRuntime | None = None
+        self._waiting_for_generated_end_speech = False
 
     async def greeting(self, cfg: AssistantConfig) -> str:
         return self._store.render(cfg.greeting) if self._dynamic_enabled else cfg.greeting
@@ -49,6 +50,7 @@ class PromptBrain(BaseBrain):
 
     async def setup(self, cfg: AssistantConfig, runtime: BrainRuntime) -> None:
         self._runtime = runtime
+        self._waiting_for_generated_end_speech = False
         schemas: list[FunctionSchema] = []
         for tool in cfg.tools:
             if tool.type == "end_call":
@@ -67,6 +69,10 @@ class PromptBrain(BaseBrain):
         self._store.record("user", content)
         self._refresh_prompt()
 
+    async def on_assistant_text_start(self, _turn_id: str) -> None:
+        if self._runtime is not None:
+            self._runtime.call_end.begin_response()
+
     async def on_assistant_text_end(
         self,
         _turn_id: str,
@@ -76,6 +82,15 @@ class PromptBrain(BaseBrain):
         if content and not interrupted:
             self._store.record("agent", content, completed_agent_turn=True)
             self._refresh_prompt()
+        if (
+            self._waiting_for_generated_end_speech
+            and self._runtime is not None
+            and self._runtime.call_end.ending
+        ):
+            self._waiting_for_generated_end_speech = False
+            await self._runtime.call_end.finish_after_current_speech(
+                has_text=bool(content.strip()) and not interrupted
+            )
 
     def _refresh_prompt(self) -> None:
         if self._dynamic_enabled and self._runtime is not None:
@@ -104,8 +119,7 @@ class PromptBrain(BaseBrain):
         )
         return schema, call_http
 
-    @staticmethod
-    def _make_end_call_tool(tool, runtime: BrainRuntime):
+    def _make_end_call_tool(self, tool, runtime: BrainRuntime):
         config = (tool.definition or {}).get("config") or {}
         message_type = str(config.get("message_type") or "none")
         custom_message = str(config.get("custom_message") or "").strip()
@@ -113,14 +127,18 @@ class PromptBrain(BaseBrain):
 
         async def end_call(params: FunctionCallParams) -> None:
             reason = str(params.arguments.get("reason") or "end_call_tool").strip()
+            uses_custom_message = message_type == "custom" and bool(custom_message)
+            self._waiting_for_generated_end_speech = not uses_custom_message
             runtime.call_end.begin(reason)
             await params.result_callback(
                 {"status": "success", "action": "ending_call"},
                 properties=FunctionCallResultProperties(run_llm=False),
             )
 
-            if message_type != "custom" or not custom_message:
-                await runtime.call_end.finish()
+            if not uses_custom_message:
+                # The model may have already streamed a spoken goodbye before
+                # invoking this tool. Decide at assistant-text-end whether to
+                # wait for that TTS audio or finish immediately for tool-only calls.
                 return
 
             turn_id = uuid4().hex

backend/services/pipecat/call_lifecycle.py

@@ -17,6 +17,7 @@ class CallEndCoordinator:
         self._ending = False
         self._armed = False
         self._speaking = False
+        self._response_speech_started = False
         self._finished = False
         self._reason = "completed"
 
@@ -28,7 +29,22 @@ class CallEndCoordinator:
         self._ending = True
         self._reason = reason or "completed"
 
+    def begin_response(self) -> None:
+        """Start tracking speech produced by one LLM response."""
+        self._response_speech_started = False
+
     def arm_after_speech(self) -> None:
+        """Wait for the next observed bot speech to finish."""
+        self._armed = True
+
+    async def finish_after_current_speech(self, *, has_text: bool) -> None:
+        """Finish now if speech is absent/done, otherwise wait for its stop."""
+        if not has_text:
+            await self.finish()
+            return
+        if self._response_speech_started and not self._speaking:
+            await self.finish()
+            return
         self._armed = True
 
     async def finish(self) -> None:
@@ -38,15 +54,14 @@ class CallEndCoordinator:
         await self._queue_end(self._reason)
 
     async def observe(self, frame) -> None:
-        if isinstance(frame, BotStartedSpeakingFrame) and self._armed:
+        if isinstance(frame, BotStartedSpeakingFrame):
             self._speaking = True
-        elif (
-            isinstance(frame, BotStoppedSpeakingFrame)
-            and self._armed
-            and self._speaking
-        ):
-            logger.info("结束语播报完毕,挂断通话")
-            await self.finish()
+            self._response_speech_started = True
+        elif isinstance(frame, BotStoppedSpeakingFrame) and self._speaking:
+            self._speaking = False
+            if self._armed:
+                logger.info("结束语播报完毕,挂断通话")
+                await self.finish()
 
 
 class EndCallAfterSpeechProcessor(FrameProcessor):