"""Qwen-Audio Realtime speech-to-speech service for Pipecat. The adapter intentionally depends only on Pipecat's public frame/service APIs. Provider-specific WebSocket events stay in this module so the shared pipeline does not need to know about Qwen-Audio protocol details. """ from __future__ import annotations import asyncio import base64 import inspect import json from collections.abc import Awaitable, Callable from typing import Any from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit from uuid import uuid4 from loguru import logger from pipecat.frames.frames import ( CancelFrame, EndFrame, Frame, InputAudioRawFrame, InterruptionFrame, LLMMessagesAppendFrame, OutputTransportMessageUrgentFrame, StartFrame, TTSAudioRawFrame, ) from pipecat.processors.frame_processor import FrameDirection from pipecat.services.ai_service import AIService from pipecat.services.settings import ServiceSettings from pipecat.utils.time import time_now_iso8601 from websockets.asyncio.client import connect as websocket_connect from websockets.protocol import State from services.pipecat.realtime_tools import ( RealtimeTool, RealtimeToolDispatcher, RealtimeToolSession, ) DEFAULT_QWEN_AUDIO_REALTIME_MODEL = "qwen-audio-3.0-realtime-flash" DEFAULT_QWEN_AUDIO_REALTIME_VOICE = "longanqian" QWEN_INPUT_SAMPLE_RATE = 16_000 QWEN_OUTPUT_SAMPLE_RATE = 24_000 SUPPORTED_TURN_DETECTION_MODES = frozenset({"server_vad", "smart_turn"}) ExtraEventHandler = Callable[[dict[str, Any]], Awaitable[None] | None] SpeechStartedHandler = Callable[[], Awaitable[None]] class QwenAudioRealtimeService(AIService): """Translate Pipecat frames to Qwen-Audio Realtime WebSocket events. ``extra_event_handlers`` is deliberately small: future features such as Function Calling can subscribe to provider events without changing the audio, transcript, or interruption paths implemented here. """ def __init__( self, *, api_key: str, model: str = DEFAULT_QWEN_AUDIO_REALTIME_MODEL, base_url: str, instructions: str = "", voice: str = DEFAULT_QWEN_AUDIO_REALTIME_VOICE, input_sample_rate: int = QWEN_INPUT_SAMPLE_RATE, output_sample_rate: int = QWEN_OUTPUT_SAMPLE_RATE, turn_detection_mode: str = "server_vad", vad_threshold: float = 0.5, silence_duration_ms: int = 800, max_history_turns: int = 20, extra_event_handlers: dict[str, ExtraEventHandler] | None = None, **kwargs, ) -> None: if turn_detection_mode not in SUPPORTED_TURN_DETECTION_MODES: supported = ", ".join(sorted(SUPPORTED_TURN_DETECTION_MODES)) raise ValueError( f"Unsupported Qwen turn detection mode {turn_detection_mode!r}; " f"expected one of: {supported}" ) if not -1.0 <= vad_threshold <= 1.0: raise ValueError("Qwen VAD threshold must be between -1.0 and 1.0") if not 200 <= silence_duration_ms <= 6_000: raise ValueError( "Qwen VAD silence duration must be between 200 and 6000 ms" ) if not 1 <= max_history_turns <= 50: raise ValueError("Qwen max history turns must be between 1 and 50") super().__init__(settings=ServiceSettings(model=model), **kwargs) self._api_key = api_key self._model = model self._base_url = base_url self._instructions = instructions self._voice = voice self._input_sample_rate = input_sample_rate self._output_sample_rate = output_sample_rate self._turn_detection_mode = turn_detection_mode self._vad_threshold = vad_threshold self._silence_duration_ms = silence_duration_ms self._max_history_turns = max_history_turns self._extra_event_handlers = extra_event_handlers or {} self._websocket = None self._receive_task: asyncio.Task | None = None self._session_ready = asyncio.Event() self._pending_events: list[dict[str, Any]] = [] self._warned_input_sample_rate = False self._response_active = False self._audio_suppressed = False self._assistant_turn_id: str | None = None self._assistant_text = "" self._assistant_timestamp = "" self._greeting_request_item_id: str | None = None self._user_transcript_pending = False self._user_transcript_item_id = "" self._user_transcript_timestamp = "" self._deferred_assistant_messages: list[dict[str, Any]] = [] self._tools: list[RealtimeTool] = [] self._tool_session = RealtimeToolSession(self._send_tool_event) self._fixed_speech_completion: asyncio.Future[None] | None = None self._suppress_response_transcript = False self._speech_started_handler: SpeechStartedHandler | None = None self._function_names: dict[str, str] = {} async def start(self, frame: StartFrame) -> None: await super().start(frame) if not self._api_key or not self._model or not self._base_url: await self.push_error( "Qwen-Audio Realtime requires api_key, model, and base_url", fatal=True, ) return await self._connect() async def stop(self, frame: EndFrame) -> None: await self._disconnect() await super().stop(frame) async def cancel(self, frame: CancelFrame) -> None: await self._disconnect() await super().cancel(frame) async def cleanup(self) -> None: await self._disconnect() await super().cleanup() async def process_frame(self, frame: Frame, direction: FrameDirection) -> None: await super().process_frame(frame, direction) if isinstance(frame, InputAudioRawFrame): if ( frame.sample_rate != self._input_sample_rate and not self._warned_input_sample_rate ): self._warned_input_sample_rate = True logger.warning( "Qwen-Audio Realtime expected {} Hz input, received {} Hz", self._input_sample_rate, frame.sample_rate, ) await self._send_event( { "type": "input_audio_buffer.append", "audio": base64.b64encode(frame.audio).decode("ascii"), } ) return if isinstance(frame, LLMMessagesAppendFrame): for message in frame.messages: text = self._message_text(message) if text: await self.send_text( text, run_immediately=frame.run_llm is not False, ) return if isinstance(frame, InterruptionFrame): await self._cancel_active_response() await self.push_frame(frame, direction) async def send_text(self, text: str, *, run_immediately: bool = True) -> None: """Append text to Qwen's conversation and optionally start a response.""" if not text: return await self._send_event( { "type": "conversation.item.create", "item": { "type": "message", "role": "user", "content": [{"type": "input_text", "text": text}], }, } ) if run_immediately: await self._send_event({"type": "response.create"}) async def interrupt(self) -> None: """Cancel current model output and notify the rest of the pipeline.""" await self._cancel_active_response() await self.broadcast_interruption() async def request_response(self) -> None: await self._send_event({"type": "response.create"}) def set_speech_started_handler( self, handler: SpeechStartedHandler | None, ) -> None: self._speech_started_handler = handler async def speak(self, text: str) -> None: """Ask Qwen to speak a fixed greeting, then remove the hidden request. Qwen's documented ``response.create`` payload has no per-response instruction field. A temporary user item keeps this behavior within the supported protocol; it is deleted after the response completes. """ await self.speak_fixed(text, suppress_transcript=False) async def speak_fixed( self, text: str, *, suppress_transcript: bool = True, ) -> Awaitable[None] | None: """Speak configured text and expose the provider response boundary.""" if not text: return None completion = asyncio.get_running_loop().create_future() self._resolve_fixed_speech() self._fixed_speech_completion = completion self._suppress_response_transcript = suppress_transcript item_id = f"item_{uuid4().hex}" self._greeting_request_item_id = item_id await self._send_event( { "type": "conversation.item.create", "item": { "id": item_id, "type": "message", "role": "user", "content": [ { "type": "input_text", "text": ( "请直接朗读下面的开场白，不要增删、解释或添加前后缀：\n" f"{text}" ), } ], }, } ) await self._send_event({"type": "response.create"}) return completion async def update_instructions(self, instructions: str) -> None: """Update only instructions after startup. Qwen accepts voice and turn detection only in the first session update, so resending the full initial configuration would break live dynamic variable updates. """ self._instructions = instructions if self._session_ready.is_set(): await self._send_event( { "type": "session.update", "session": {"instructions": instructions}, }, wait_until_ready=False, ) async def update_session( self, instructions: str, tools: list[RealtimeTool], ) -> None: """Atomically replace the active Workflow prompt and tool catalog.""" self._instructions = instructions self._tools = list(tools) if self._session_ready.is_set(): await self._send_event( { "type": "session.update", "session": { "instructions": instructions, "tools": [tool.provider_schema() for tool in tools], "tool_choice": "auto", }, }, wait_until_ready=False, ) def set_tool_dispatcher( self, dispatcher: RealtimeToolDispatcher | None, ) -> None: self._tool_session.set_dispatcher(dispatcher) def _connection_url(self) -> str: parts = urlsplit(self._base_url) query = dict(parse_qsl(parts.query)) query["model"] = self._model return urlunsplit( (parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment) ) def _initial_session_config(self) -> dict[str, Any]: return { "modalities": ["text", "audio"], "instructions": self._instructions, "voice": self._voice, "input_audio_format": "pcm", "output_audio_format": "pcm", "turn_detection": self._turn_detection_config(), "max_history_turns": self._max_history_turns, "tools": [tool.provider_schema() for tool in self._tools], "tool_choice": "auto", } def _turn_detection_config(self) -> dict[str, Any]: if self._turn_detection_mode == "smart_turn": return {"type": "smart_turn"} return { "type": "server_vad", "threshold": self._vad_threshold, "silence_duration_ms": self._silence_duration_ms, } async def _connect(self) -> None: if self._websocket and self._websocket.state is State.OPEN: return try: self._websocket = await websocket_connect( self._connection_url(), additional_headers={ "Authorization": f"Bearer {self._api_key}", "x-dashscope-dataInspection": "disable", }, max_size=None, open_timeout=10, ) self._receive_task = self.create_task( self._receive_messages(), name="qwen_audio_realtime_receive" ) except Exception as exc: self._websocket = None await self.push_error( f"Qwen-Audio Realtime connection failed: {exc}", exception=exc, fatal=True, ) async def _disconnect(self) -> None: current_task = asyncio.current_task() task = self._receive_task self._receive_task = None if task and task is not current_task: await self.cancel_task(task) websocket = self._websocket self._websocket = None self._session_ready.clear() self._pending_events.clear() self._response_active = False self._user_transcript_pending = False self._user_transcript_item_id = "" self._user_transcript_timestamp = "" self._deferred_assistant_messages.clear() self._tool_session.clear() self._function_names.clear() self._resolve_fixed_speech() if websocket and websocket.state is State.OPEN: try: await websocket.close() except Exception: pass async def _receive_messages(self) -> None: websocket = self._websocket if not websocket: return try: async for raw_message in websocket: await self._handle_server_event(json.loads(raw_message)) except Exception as exc: if self._websocket is websocket: await self.push_error( f"Qwen-Audio Realtime receive failed: {exc}", exception=exc ) finally: if self._websocket is websocket: self._websocket = None self._session_ready.clear() if self._receive_task is asyncio.current_task(): self._receive_task = None async def _handle_server_event(self, event: dict[str, Any]) -> None: event_type = str(event.get("type") or "") if event_type == "session.created": await self._send_event( { "type": "session.update", "session": self._initial_session_config(), }, wait_until_ready=False, ) elif event_type == "session.updated": self._session_ready.set() pending, self._pending_events = self._pending_events, [] for payload in pending: await self._send_event(payload, wait_until_ready=False) elif event_type == "response.created": self._response_active = True self._audio_suppressed = False elif event_type == "response.audio.delta": audio = event.get("delta") if audio and not self._audio_suppressed: await self.push_frame( TTSAudioRawFrame( base64.b64decode(audio), self._output_sample_rate, 1, ) ) elif event_type in {"response.audio_transcript.delta", "response.text.delta"}: if not self._audio_suppressed and not self._suppress_response_transcript: await self._append_assistant_text(str(event.get("delta") or "")) elif event_type in {"response.audio_transcript.done", "response.text.done"}: transcript = str(event.get("transcript") or event.get("text") or "") if ( transcript and not self._audio_suppressed and not self._suppress_response_transcript ): if self._assistant_turn_id: self._assistant_text = transcript else: await self._append_assistant_text(transcript) elif event_type == "conversation.item.input_audio_transcription.completed": await self._handle_user_transcript_completed(event) elif event_type == "conversation.item.input_audio_transcription.failed": await self._release_deferred_assistant_messages() elif event_type == "input_audio_buffer.speech_started": user_turn_timestamp = time_now_iso8601() await self._cancel_active_response() await self.broadcast_interruption() if self._speech_started_handler is not None: await self._speech_started_handler() await self._start_user_transcript_turn(event, user_turn_timestamp) elif ( event_type == "input_audio_buffer.speech_stopped" and event.get("reason") == "turn_invalid" ): await self._release_deferred_assistant_messages() elif event_type == "response.done": response = event.get("response") status = response.get("status") if isinstance(response, dict) else None interrupted = status in {"cancelled", "incomplete", "interrupted", "failed"} self._response_active = False await self._finish_assistant_text(interrupted=interrupted) await self._delete_greeting_request() self._resolve_fixed_speech() elif event_type == "response.output_item.added": self._remember_function_call(event) elif event_type in { "response.function_call_arguments.done", "response.output_item.done", }: await self._handle_function_call_event(event) elif event_type == "error": error = event.get("error") message = error.get("message") if isinstance(error, dict) else str(error) if "cancel" not in str(message).lower(): await self.push_error(f"Qwen-Audio Realtime error: {message}") self._resolve_fixed_speech() handler = self._extra_event_handlers.get(event_type) if handler: result = handler(event) if inspect.isawaitable(result): await result async def _cancel_active_response(self) -> None: if not self._response_active and not self._assistant_turn_id: return self._audio_suppressed = True if self._response_active: await self._send_event( {"type": "response.cancel"}, wait_until_ready=False ) self._response_active = False await self._finish_assistant_text(interrupted=True) self._resolve_fixed_speech() async def _send_tool_event(self, payload: dict[str, Any]) -> None: await self._send_event(payload, wait_until_ready=False) async def _handle_function_call_event(self, event: dict[str, Any]) -> None: item = event.get("item") source = item if isinstance(item, dict) else event if isinstance(item, dict) and item.get("type") != "function_call": return call_id = str( source.get("call_id") or event.get("call_id") or source.get("id") or "" ) name = str( source.get("name") or event.get("name") or self._function_names.get(call_id) or "" ) if not name: return await self._tool_session.handle_call( name=name, call_id=call_id, arguments=source.get("arguments", event.get("arguments")), ) self._function_names.pop(call_id, None) def _remember_function_call(self, event: dict[str, Any]) -> None: item = event.get("item") if not isinstance(item, dict) or item.get("type") != "function_call": return call_id = str(item.get("call_id") or item.get("id") or "") name = str(item.get("name") or "") if call_id and name: self._function_names[call_id] = name def _resolve_fixed_speech(self) -> None: completion = self._fixed_speech_completion self._fixed_speech_completion = None self._suppress_response_transcript = False if completion is not None and not completion.done(): completion.set_result(None) async def _delete_greeting_request(self) -> None: item_id = self._greeting_request_item_id self._greeting_request_item_id = None if item_id: await self._send_event( {"type": "conversation.item.delete", "item_id": item_id}, wait_until_ready=False, ) async def _start_user_transcript_turn( self, event: dict[str, Any], timestamp: str, ) -> None: """Hold the next assistant transcript until this user turn is visible. Qwen produces the assistant response and the final ASR transcript on independent streams. The assistant delta can therefore arrive first. Keep this provider-specific ordering rule inside the adapter so the shared cascade pipeline and Debug Drawer protocol remain unchanged. """ if self._user_transcript_pending: await self._release_deferred_assistant_messages() self._user_transcript_pending = True self._user_transcript_item_id = str(event.get("item_id") or "") self._user_transcript_timestamp = timestamp async def _handle_user_transcript_completed( self, event: dict[str, Any], ) -> None: item_id = str(event.get("item_id") or "") timestamp = ( self._user_transcript_timestamp if self._user_transcript_pending and ( not self._user_transcript_item_id or not item_id or item_id == self._user_transcript_item_id ) else "" ) await self._send_transcript( "user", str(event.get("transcript") or ""), timestamp=timestamp or None, ) if timestamp: await self._release_deferred_assistant_messages() async def _send_assistant_transport_message( self, message: dict[str, Any], ) -> None: if self._user_transcript_pending: self._deferred_assistant_messages.append(message) return await self._send_transport_message(message) async def _release_deferred_assistant_messages(self) -> None: self._user_transcript_pending = False self._user_transcript_item_id = "" self._user_transcript_timestamp = "" pending, self._deferred_assistant_messages = ( self._deferred_assistant_messages, [], ) for message in pending: await self._send_transport_message(message) async def _send_event( self, payload: dict[str, Any], *, wait_until_ready: bool = True ) -> None: if wait_until_ready and not self._session_ready.is_set(): self._pending_events.append(payload) return if not self._websocket or self._websocket.state is not State.OPEN: return event = {"event_id": f"event_{uuid4().hex}", **payload} await self._websocket.send(json.dumps(event, ensure_ascii=False)) async def _append_assistant_text(self, delta: str) -> None: if not delta: return if not self._assistant_turn_id: self._assistant_turn_id = uuid4().hex self._assistant_timestamp = time_now_iso8601() await self._send_assistant_transport_message( { "type": "assistant-text-start", "turn_id": self._assistant_turn_id, "timestamp": self._assistant_timestamp, } ) self._assistant_text += delta await self._send_assistant_transport_message( { "type": "assistant-text-delta", "turn_id": self._assistant_turn_id, "delta": delta, } ) async def _finish_assistant_text(self, *, interrupted: bool) -> None: if not self._assistant_turn_id: return await self._send_assistant_transport_message( { "type": "assistant-text-end", "turn_id": self._assistant_turn_id, "content": self._assistant_text, "interrupted": interrupted, } ) self._assistant_turn_id = None self._assistant_text = "" self._assistant_timestamp = "" async def _send_transcript( self, role: str, content: str, *, timestamp: str | None = None, ) -> None: if content: await self._send_transport_message( { "type": "transcript", "role": role, "content": content, "timestamp": timestamp or time_now_iso8601(), } ) async def _send_transport_message(self, message: dict[str, Any]) -> None: await self.push_frame(OutputTransportMessageUrgentFrame(message=message)) @staticmethod def _message_text(message: Any) -> str: if not isinstance(message, dict): return "" content = message.get("content") if isinstance(content, str): return content.strip() if isinstance(content, list): return "\n".join( str(part.get("text") or "") for part in content if isinstance(part, dict) and part.get("type") == "text" ).strip() return ""