feat: enhance call end coordination with state management

- Added support for tracking call end states (pending, speaking, cancelled) in CallEndCoordinator. - Introduced a callback mechanism for state changes to allow external handling of call end states. - Updated pipeline and tests to accommodate new state management features, ensuring proper handling of call end scenarios.
2026-08-12 14:25:24 +08:00
parent d0bb5b3868
commit 79a239489a
5 changed files with 234 additions and 12 deletions
--- a/backend/services/pipecat/call_lifecycle.py
+++ b/backend/services/pipecat/call_lifecycle.py
@@ -63,11 +63,19 @@ def playback_marker_for(
    return FixedSpeechPlaybackMarkerFrame(completion=completion)


+CallEndStateHandler = Callable[[str, bool], Awaitable[None]]
+
+
 class CallEndCoordinator:
    """Coordinate committed and user-cancellable conversation endings."""

-    def __init__(self, queue_end: Callable[[str], Awaitable[None]]):
+    def __init__(
+        self,
+        queue_end: Callable[[str], Awaitable[None]],
+        on_state_change: CallEndStateHandler | None = None,
+    ):
        self._queue_end = queue_end
+        self._on_state_change = on_state_change
        self._ending = False
        self._interruptible = False
        self._armed = False
@@ -110,6 +118,7 @@ class CallEndCoordinator:
        self._armed = False
        self._finish_after_tracked_speech = False
        self._reason = reason or "completed"
+        self._schedule_state("speaking" if self._speaking else "pending")

    def cancel_pending_end(self) -> bool:
        """Cancel one pending ending when its tool explicitly allows it."""
@@ -123,8 +132,13 @@ class CallEndCoordinator:
        # Transport speech state remains authoritative: if audio is currently
        # playing, the interruption path will still emit BotStoppedSpeakingFrame.
        logger.info("用户打断结束语,已撤销待挂断状态")
+        self._schedule_state("cancelled")
        return True

+    def _schedule_state(self, state: str) -> None:
+        if self._on_state_change is not None:
+            asyncio.create_task(self._on_state_change(state, self.interruptible))
+
    def begin_response(self) -> None:
        """Start tracking speech produced by one LLM response."""
        self._response_speech_started = False
@@ -200,6 +214,8 @@ class CallEndCoordinator:
            self._speaking = True
            self._speech_stopped.clear()
            self._response_speech_started = True
+            if self._ending:
+                await self._emit_state("speaking")
        elif isinstance(frame, BotStoppedSpeakingFrame) and self._speaking:
            self._speaking = False
            self._speech_stopped.set()
@@ -207,6 +223,10 @@ class CallEndCoordinator:
                logger.info("结束语播报完毕,挂断通话")
                await self.finish()

+    async def _emit_state(self, state: str) -> None:
+        if self._on_state_change is not None:
+            await self._on_state_change(state, self.interruptible)
+

 class EndCallAfterSpeechProcessor(FrameProcessor):
    def __init__(self, coordinator: CallEndCoordinator):
--- a/backend/services/pipecat/pipeline.py
+++ b/backend/services/pipecat/pipeline.py
@@ -201,6 +201,23 @@ def _multimodal_user_input_frame(
    )


+def _call_end_state_frame(
+    state: str,
+    *,
+    interruptible: bool,
+    voice_interruptible: bool,
+) -> OutputTransportMessageUrgentFrame:
+    """Build the client event used to render the pending call-end state."""
+    return OutputTransportMessageUrgentFrame(
+        message={
+            "type": "call-end-state",
+            "state": state,
+            "interruptible": interruptible,
+            "voice_interruptible": voice_interruptible,
+        }
+    )
+
+
 async def run_pipeline(
    transport,
    cfg: AssistantConfig,
@@ -295,7 +312,24 @@ async def run_pipeline(
        )
        await worker.queue_frame(EndFrame())

-    call_end = CallEndCoordinator(queue_call_end)
+    async def queue_call_end_state(state: str, interruptible: bool) -> None:
+        worker = worker_holder.get("worker")
+        if worker is None:
+            return
+        await worker.queue_frame(
+            _call_end_state_frame(
+                state,
+                interruptible=interruptible,
+                voice_interruptible=(
+                    interruptible and current_enable_interrupt
+                ),
+            )
+        )
+
+    call_end = CallEndCoordinator(
+        queue_call_end,
+        on_state_change=queue_call_end_state,
+    )

    knowledge_config = cfg.knowledge_retrieval_config
    knowledge_mode = str(knowledge_config.get("mode", "automatic"))
@@ -976,7 +1010,24 @@ async def run_realtime_pipeline(
        )
        await worker.queue_frame(EndFrame())

-    call_end = CallEndCoordinator(queue_call_end)
+    async def queue_call_end_state(state: str, interruptible: bool) -> None:
+        worker = worker_holder.get("worker")
+        if worker is None:
+            return
+        await worker.queue_frame(
+            _call_end_state_frame(
+                state,
+                interruptible=interruptible,
+                voice_interruptible=(
+                    interruptible and cfg.enableInterrupt
+                ),
+            )
+        )
+
+    call_end = CallEndCoordinator(
+        queue_call_end,
+        on_state_change=queue_call_end_state,
+    )
    client_tools = ClientToolBroker()

    async def interrupt_for_client_tool_result() -> None:
--- a/backend/tests/test_call_lifecycle.py
+++ b/backend/tests/test_call_lifecycle.py
@@ -19,20 +19,26 @@ from services.pipecat.processors import CallEndingUserMuteStrategy
 class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
    async def asyncSetUp(self):
        self.reasons: list[str] = []
+        self.states: list[tuple[str, bool]] = []

        async def queue_end(reason: str) -> None:
            self.reasons.append(reason)

-        self.coordinator = CallEndCoordinator(queue_end)
+        async def record_state(state: str, interruptible: bool) -> None:
+            self.states.append((state, interruptible))
+
+        self.coordinator = CallEndCoordinator(queue_end, record_state)

    async def test_generated_closing_text_waits_for_audio_stop(self):
        self.coordinator.begin_response()
        self.coordinator.begin("prompt_end_call")
+        await asyncio.sleep(0)
        await self.coordinator.finish_after_current_speech(has_text=True)

        self.assertEqual(self.reasons, [])
        await self.coordinator.observe(BotStartedSpeakingFrame())
        self.assertEqual(self.reasons, [])
+        self.assertEqual(self.states[-1], ("speaking", False))
        await self.coordinator.observe(BotStoppedSpeakingFrame())
        self.assertEqual(self.reasons, ["prompt_end_call"])

@@ -71,12 +77,17 @@ class CallEndCoordinatorTest(unittest.IsolatedAsyncioTestCase):
    async def test_interruptible_end_keeps_input_open_and_can_be_cancelled(self):
        self.coordinator.begin_response()
        self.coordinator.begin("user_goodbye", interruptible=True)
+        await asyncio.sleep(0)
+        self.assertEqual(self.states[-1], ("pending", True))
        await self.coordinator.finish_after_current_speech(has_text=True)
        await self.coordinator.observe(BotStartedSpeakingFrame())
+        self.assertEqual(self.states[-1], ("speaking", True))

        self.assertTrue(self.coordinator.ending)
        self.assertTrue(self.coordinator.accepts_input)
        self.assertTrue(self.coordinator.cancel_pending_end())
+        await asyncio.sleep(0)
+        self.assertEqual(self.states[-1], ("cancelled", False))
        self.assertFalse(self.coordinator.ending)
        self.assertTrue(self.coordinator.accepts_input)
        self.assertTrue(self.coordinator.speaking)
--- a/frontend/src/components/assistant-editor/debug-preview.tsx
+++ b/frontend/src/components/assistant-editor/debug-preview.tsx
@@ -58,6 +58,7 @@ import {
  createUserInputId,
  useVoicePreview,
  type ChatMessage,
+  type CallEndState,
  type ClientToolDefinition,
  type UserInputPart,
  type VoicePreview,
@@ -273,6 +274,7 @@ export function DebugDrawer({
              <DebugConnectionStatus
                status={preview.status}
                micWarning={preview.micWarning}
+                callEndState={preview.callEndState}
              />
            </>
          )}
@@ -724,6 +726,7 @@ function DeviceSelectField({
  value,
  devices,
  onSelect,
+  disabled = false,
 }: {
  icon: React.ReactNode;
  ariaLabel: string;
@@ -732,9 +735,11 @@ function DeviceSelectField({
  value: string;
  devices: MediaDeviceInfo[];
  onSelect: (deviceId: string) => void;
+  disabled?: boolean;
 }) {
  return (
    <Select
+      disabled={disabled}
      value={value || "default"}
      onValueChange={(nextValue) =>
        onSelect(nextValue === "default" ? "" : nextValue)
@@ -769,7 +774,13 @@ function DeviceSelectField({
  );
 }

-function MicrophoneDeviceField({ preview }: { preview: VoicePreview }) {
+function MicrophoneDeviceField({
+  preview,
+  disabled = false,
+}: {
+  preview: VoicePreview;
+  disabled?: boolean;
+}) {
  return (
    <DeviceSelectField
      icon={<Mic size={15} className="shrink-0 text-muted-soft" />}
@@ -779,6 +790,7 @@ function MicrophoneDeviceField({ preview }: { preview: VoicePreview }) {
      value={preview.selectedDeviceId}
      devices={preview.audioInputs}
      onSelect={(deviceId) => preview.selectDevice(deviceId)}
+      disabled={disabled}
    />
  );
 }
@@ -808,11 +820,13 @@ function DebugInputModeButton({
  label,
  onClick,
  children,
+  disabled = false,
 }: {
  selected: boolean;
  label: string;
  onClick: () => void;
  children: React.ReactNode;
+  disabled?: boolean;
 }) {
  return (
    <button
@@ -820,12 +834,14 @@ function DebugInputModeButton({
      aria-label={label}
      aria-pressed={selected}
      title={label}
+      disabled={disabled}
      onClick={onClick}
      className={[
        "flex h-8 w-8 shrink-0 items-center justify-center rounded-full transition-colors",
        selected
          ? "bg-surface-strong text-foreground"
          : "text-muted-soft hover:bg-surface-strong hover:text-foreground",
+        disabled ? "cursor-not-allowed" : "",
      ].join(" ")}
    >
      {children}
@@ -875,6 +891,12 @@ function DebugVoicePanel({
    audioRef,
  } = preview;
  const recording = status === "connecting" || status === "connected";
+  const callEndActive =
+    preview.callEndState.phase === "pending" ||
+    preview.callEndState.phase === "speaking";
+  const canInterruptEnd =
+    callEndActive && preview.callEndState.interruptible;
+  const endInputLocked = callEndActive && !preview.callEndState.interruptible;
  const [textDraft, setTextDraft] = useState("");
  const [inputMode, setInputMode] = useState<DebugInputMode>("mic");
  const [pendingImage, setPendingImage] =
@@ -1181,6 +1203,30 @@ function DebugVoicePanel({
      </div>

      <div className="shrink-0 border-t border-hairline bg-card p-3">
+        {callEndActive && (
+          <div
+            role="status"
+            aria-live="polite"
+            className="mb-2 flex min-w-0 items-center gap-2 px-2 text-[11px] leading-4 text-muted-foreground"
+          >
+            <span
+              aria-hidden
+              className={[
+                "h-1.5 w-1.5 shrink-0 rounded-full",
+                canInterruptEnd
+                  ? "animate-pulse bg-gradient-peach"
+                  : "bg-muted-soft",
+              ].join(" ")}
+            />
+            <span className="truncate">
+              {canInterruptEnd
+                ? preview.callEndState.voiceInterruptible
+                  ? "直接说话或发送文字，可以继续对话"
+                  : "发送文字可以继续对话；当前未开启语音打断"
+                : "输入已暂停，结束语播放完成后将结束对话"}
+            </span>
+          </div>
+        )}
        <div className="flex items-end gap-2">
          <div className="min-w-0 flex-1">
            <div
@@ -1191,7 +1237,10 @@ function DebugVoicePanel({
                "relative flex min-h-10 min-w-0 items-end gap-1 overflow-hidden rounded-[1.4rem] border bg-background px-2 transition-colors",
                draggingImage
                  ? "border-foreground"
-                  : "border-hairline-strong",
+                  : canInterruptEnd
+                    ? "border-gradient-peach shadow-[0_0_0_2px_color-mix(in_srgb,var(--gradient-peach)_16%,transparent)]"
+                    : "border-hairline-strong",
+                endInputLocked ? "pointer-events-none opacity-55" : "",
              ].join(" ")}
            >
              <div className="mb-1 flex shrink-0 items-center gap-0.5 rounded-full bg-canvas-soft p-0.5">
@@ -1199,6 +1248,7 @@ function DebugVoicePanel({
                  selected={inputMode === "mic"}
                  label="选择麦克风设备"
                  onClick={() => setInputMode("mic")}
+                  disabled={endInputLocked}
                >
                  <Mic size={15} />
                </DebugInputModeButton>
@@ -1206,13 +1256,17 @@ function DebugVoicePanel({
                  selected={inputMode === "text"}
                  label="文字输入"
                  onClick={() => setInputMode("text")}
+                  disabled={endInputLocked}
                >
                  <MessageSquareText size={15} />
                </DebugInputModeButton>
              </div>
              {inputMode === "mic" ? (
                <div className="flex min-w-0 flex-1">
-                  <MicrophoneDeviceField preview={preview} />
+                  <MicrophoneDeviceField
+                    preview={preview}
+                    disabled={endInputLocked}
+                  />
                </div>
              ) : (
                <div className="min-w-0 flex-1 py-1">
@@ -1254,7 +1308,9 @@ function DebugVoicePanel({
                  <Textarea
                    rows={1}
                    value={textDraft}
-                    disabled={status !== "connected" || sendingInput}
+                    disabled={
+                      status !== "connected" || sendingInput || endInputLocked
+                    }
                    onChange={(event) => {
                      setTextDraft(event.target.value);
                      if (inputError) setInputError("");
@@ -1272,7 +1328,9 @@ function DebugVoicePanel({
                    }}
                    placeholder={
                      status === "connected"
-                        ? vision
+                        ? canInterruptEnd
+                          ? "输入内容以继续对话…"
+                          : vision
                          ? "输入文字，或粘贴 / 拖入图片…"
                          : "输入文字发送给助手，将打断当前播报…"
                        : "开始对话后可输入文字…"
@@ -1302,6 +1360,7 @@ function DebugVoicePanel({
              disabled={
                status !== "connected" ||
                sendingInput ||
+                endInputLocked ||
                (!textDraft.trim() && !pendingImage)
              }
              onClick={() => void handleSendInput()}
@@ -1661,13 +1720,27 @@ function DebugVideoPanel({
 function DebugConnectionStatus({
  status,
  micWarning,
+  callEndState,
 }: {
  status: VoicePreviewStatus;
  micWarning: string | null;
+  callEndState: CallEndState;
 }) {
  const recording = status === "connecting" || status === "connected";
+  const ending =
+    callEndState.phase === "pending" || callEndState.phase === "speaking";
  const label =
-    status === "connecting"
+    status === "connected" && callEndState.phase === "cancelled"
+      ? "已继续对话"
+      : status === "connected" && ending
+        ? callEndState.phase === "pending"
+          ? callEndState.interruptible
+            ? "准备结束 · 可打断"
+            : "准备结束"
+          : callEndState.interruptible
+            ? "结束语播放中 · 可打断"
+            : "结束语播放中 · 即将结束"
+    : status === "connecting"
      ? "连接中…"
      : status === "connected"
        ? micWarning
@@ -1678,11 +1751,19 @@ function DebugConnectionStatus({
          : "准备开始";

  return (
-    <span className="flex min-w-0 items-center gap-1.5 truncate text-xs text-muted-foreground">
+    <span
+      role="status"
+      aria-live="polite"
+      className="flex min-w-0 items-center gap-1.5 truncate text-xs text-muted-foreground"
+    >
      <span
        className={[
          "h-1.5 w-1.5 shrink-0 rounded-full",
-          recording
+          status === "connected" && ending && callEndState.interruptible
+            ? "animate-pulse bg-gradient-peach"
+            : status === "connected" && callEndState.phase === "cancelled"
+              ? "bg-success"
+          : recording
            ? "animate-pulse bg-success"
            : status === "failed"
              ? "bg-destructive"
--- a/frontend/src/hooks/use-voice-preview.ts
+++ b/frontend/src/hooks/use-voice-preview.ts
@@ -17,9 +17,16 @@ import { API_BASE, webrtcApi } from "@/lib/api";

 export type VoicePreviewStatus = "idle" | "connecting" | "connected" | "failed";
 export type NetworkQuality = "unknown" | "good" | "fair" | "poor";
+export type CallEndPhase = "idle" | "pending" | "speaking" | "cancelled";
+export type CallEndState = {
+  phase: CallEndPhase;
+  interruptible: boolean;
+  voiceInterruptible: boolean;
+};

 const NETWORK_SAMPLE_INTERVAL_MS = 2_000;
 const SESSION_UPDATE_TIMEOUT_MS = 10_000;
+const CALL_END_CANCELLED_FEEDBACK_MS = 1_500;

 type NetworkStatsSample = {
  packetsSent: number;
@@ -293,6 +300,11 @@ export function useVoicePreview(
    Record<string, DynamicVariableValue>
  >({});
  const [callEnded, setCallEnded] = useState(false);
+  const [callEndState, setCallEndState] = useState<CallEndState>({
+    phase: "idle",
+    interruptible: false,
+    voiceInterruptible: false,
+  });
  const [networkQuality, setNetworkQuality] =
    useState<NetworkQuality>("unknown");
  const [audioInputs, setAudioInputs] = useState<MediaDeviceInfo[]>([]);
@@ -318,6 +330,7 @@ export function useVoicePreview(
    new Map<string, RegisteredClientTool>(),
  );
  const endedByServerRef = useRef(false);
+  const callEndFeedbackTimeoutRef = useRef<number | null>(null);
  const selectedDeviceIdRef = useRef("");
  const selectedOutputDeviceIdRef = useRef("");
  const onNodeActiveRef = useRef(onNodeActive);
@@ -387,6 +400,10 @@ export function useVoicePreview(
    });
    pendingSessionUpdatesRef.current.clear();
    networkStatsRef.current = null;
+    if (callEndFeedbackTimeoutRef.current !== null) {
+      window.clearTimeout(callEndFeedbackTimeoutRef.current);
+      callEndFeedbackTimeoutRef.current = null;
+    }
    onNodeActiveRef.current?.(null);
  }, []);

@@ -401,6 +418,11 @@ export function useVoicePreview(
    setError(null);
    setMicWarning(null);
    setNetworkQuality("unknown");
+    setCallEndState({
+      phase: "idle",
+      interruptible: false,
+      voiceInterruptible: false,
+    });
    setStatus("idle");
  }, [releaseResources]);

@@ -411,6 +433,11 @@ export function useVoicePreview(
    setRemoteStream(null);
    setError(message);
    setNetworkQuality("unknown");
+    setCallEndState({
+      phase: "idle",
+      interruptible: false,
+      voiceInterruptible: false,
+    });
    setStatus("failed");
  }, [releaseResources]);

@@ -620,6 +647,32 @@ export function useVoicePreview(
      setSessionVariables(publicVariableSnapshot(msg.variables));
    } else if (msg.type === "session-variables") {
      setSessionVariables(publicVariableSnapshot(msg.variables));
+    } else if (
+      msg.type === "call-end-state" &&
+      (msg.state === "pending" ||
+        msg.state === "speaking" ||
+        msg.state === "cancelled")
+    ) {
+      if (callEndFeedbackTimeoutRef.current !== null) {
+        window.clearTimeout(callEndFeedbackTimeoutRef.current);
+        callEndFeedbackTimeoutRef.current = null;
+      }
+      const phase = msg.state as Exclude<CallEndPhase, "idle">;
+      setCallEndState({
+        phase,
+        interruptible: msg.interruptible === true,
+        voiceInterruptible: msg.voice_interruptible === true,
+      });
+      if (phase === "cancelled") {
+        callEndFeedbackTimeoutRef.current = window.setTimeout(() => {
+          setCallEndState({
+            phase: "idle",
+            interruptible: false,
+            voiceInterruptible: false,
+          });
+          callEndFeedbackTimeoutRef.current = null;
+        }, CALL_END_CANCELLED_FEEDBACK_MS);
+      }
    } else if (msg.type === "call-ended") {
      endedByServerRef.current = true;
      setCallEnded(true);
@@ -645,6 +698,11 @@ export function useVoicePreview(
    setSessionVariables(publicVariableSnapshot(options.dynamicVariables ?? {}));
    pendingAssistantTurnsRef.current.clear();
    setCallEnded(false);
+    setCallEndState({
+      phase: "idle",
+      interruptible: false,
+      voiceInterruptible: false,
+    });
    endedByServerRef.current = false;

    await cleanupPromiseRef.current;
@@ -1000,6 +1058,7 @@ export function useVoicePreview(
    sessionVariables,
    clientTools,
    callEnded,
+    callEndState,
    networkQuality,
    audioInputs,
    audioOutputs,