feat(qwen-audio): add Alibaba Cloud Qwen-Audio Realtime service integration
- Introduced Qwen-Audio Realtime service for speech-to-speech processing in Pipecat. - Updated interface catalog to include Qwen-Audio Realtime capabilities. - Enhanced model resource testing to support new service. - Added configuration options for audio sample rates and turn detection modes. - Updated documentation to reflect integration details and usage instructions.
This commit is contained in:
Xin Wang
234
backend/tests/test_qwen_audio_realtime.py
Normal file
234
backend/tests/test_qwen_audio_realtime.py
Normal file
@@ -0,0 +1,234 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import unittest
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from models import AssistantConfig
|
||||
from pipecat.frames.frames import OutputTransportMessageUrgentFrame, TTSAudioRawFrame
|
||||
from services import model_resource_tester
|
||||
from services.interface_catalog import INTERFACE_DEFINITIONS
|
||||
from services.pipecat.qwen_audio_realtime import QwenAudioRealtimeService
|
||||
from services.pipecat.service_factory import (
|
||||
create_realtime_service,
|
||||
realtime_audio_sample_rates,
|
||||
)
|
||||
from websockets.protocol import State
|
||||
|
||||
|
||||
class _OpenWebSocket:
|
||||
state = State.OPEN
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.messages: list[dict] = []
|
||||
|
||||
async def send(self, raw_message: str) -> None:
|
||||
self.messages.append(json.loads(raw_message))
|
||||
|
||||
|
||||
def _service(**overrides) -> QwenAudioRealtimeService:
|
||||
settings = {
|
||||
"api_key": "test-key",
|
||||
"model": "qwen-audio-3.0-realtime-flash",
|
||||
"base_url": (
|
||||
"wss://workspace.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime"
|
||||
),
|
||||
"instructions": "You are helpful.",
|
||||
}
|
||||
settings.update(overrides)
|
||||
return QwenAudioRealtimeService(**settings)
|
||||
|
||||
|
||||
class QwenAudioRealtimeServiceTest(unittest.IsolatedAsyncioTestCase):
|
||||
def test_provider_defaults_and_session_config(self):
|
||||
service = _service()
|
||||
|
||||
self.assertEqual(
|
||||
service._connection_url(),
|
||||
"wss://workspace.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime"
|
||||
"?model=qwen-audio-3.0-realtime-flash",
|
||||
)
|
||||
self.assertEqual(
|
||||
service._initial_session_config(),
|
||||
{
|
||||
"modalities": ["text", "audio"],
|
||||
"instructions": "You are helpful.",
|
||||
"voice": "longanqian",
|
||||
"input_audio_format": "pcm",
|
||||
"output_audio_format": "pcm",
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
"threshold": 0.5,
|
||||
"silence_duration_ms": 800,
|
||||
},
|
||||
"max_history_turns": 20,
|
||||
},
|
||||
)
|
||||
|
||||
def test_smart_turn_and_value_ranges_are_explicit(self):
|
||||
service = _service(turn_detection_mode="smart_turn")
|
||||
self.assertEqual(service._turn_detection_config(), {"type": "smart_turn"})
|
||||
|
||||
with self.assertRaisesRegex(ValueError, "turn detection mode"):
|
||||
_service(turn_detection_mode="manual")
|
||||
with self.assertRaisesRegex(ValueError, "threshold"):
|
||||
_service(vad_threshold=1.1)
|
||||
with self.assertRaisesRegex(ValueError, "history"):
|
||||
_service(max_history_turns=51)
|
||||
|
||||
async def test_session_ready_flushes_queued_events_in_order(self):
|
||||
service = _service()
|
||||
websocket = _OpenWebSocket()
|
||||
service._websocket = websocket
|
||||
|
||||
await service.send_text("hello")
|
||||
self.assertEqual(websocket.messages, [])
|
||||
|
||||
await service._handle_server_event({"type": "session.created"})
|
||||
self.assertEqual(websocket.messages[0]["type"], "session.update")
|
||||
self.assertEqual(
|
||||
websocket.messages[0]["session"]["input_audio_format"], "pcm"
|
||||
)
|
||||
|
||||
await service._handle_server_event({"type": "session.updated"})
|
||||
self.assertEqual(
|
||||
[message["type"] for message in websocket.messages],
|
||||
["session.update", "conversation.item.create", "response.create"],
|
||||
)
|
||||
|
||||
async def test_dynamic_instructions_do_not_resend_locked_settings(self):
|
||||
service = _service()
|
||||
websocket = _OpenWebSocket()
|
||||
service._websocket = websocket
|
||||
service._session_ready.set()
|
||||
|
||||
await service.update_instructions("Call the user Alice.")
|
||||
|
||||
self.assertEqual(websocket.messages[-1]["type"], "session.update")
|
||||
self.assertEqual(
|
||||
websocket.messages[-1]["session"],
|
||||
{"instructions": "Call the user Alice."},
|
||||
)
|
||||
|
||||
async def test_interruption_cancels_and_suppresses_residual_audio(self):
|
||||
service = _service()
|
||||
websocket = _OpenWebSocket()
|
||||
service._websocket = websocket
|
||||
service._session_ready.set()
|
||||
service.push_frame = AsyncMock()
|
||||
service.broadcast_interruption = AsyncMock()
|
||||
audio = base64.b64encode(b"\x01\x02").decode("ascii")
|
||||
|
||||
await service._handle_server_event({"type": "response.created"})
|
||||
await service._handle_server_event(
|
||||
{"type": "response.audio_transcript.delta", "delta": "Hi"}
|
||||
)
|
||||
await service._handle_server_event(
|
||||
{"type": "response.audio.delta", "delta": audio}
|
||||
)
|
||||
await service._handle_server_event(
|
||||
{"type": "input_audio_buffer.speech_started"}
|
||||
)
|
||||
calls_before_residual_audio = service.push_frame.await_count
|
||||
await service._handle_server_event(
|
||||
{"type": "response.audio.delta", "delta": audio}
|
||||
)
|
||||
|
||||
self.assertEqual(service.push_frame.await_count, calls_before_residual_audio)
|
||||
self.assertIn("response.cancel", [item["type"] for item in websocket.messages])
|
||||
service.broadcast_interruption.assert_awaited_once()
|
||||
|
||||
frames = [call.args[0] for call in service.push_frame.await_args_list]
|
||||
self.assertTrue(any(isinstance(frame, TTSAudioRawFrame) for frame in frames))
|
||||
end_messages = [
|
||||
frame.message
|
||||
for frame in frames
|
||||
if isinstance(frame, OutputTransportMessageUrgentFrame)
|
||||
and frame.message.get("type") == "assistant-text-end"
|
||||
]
|
||||
self.assertEqual(end_messages[0]["content"], "Hi")
|
||||
self.assertTrue(end_messages[0]["interrupted"])
|
||||
|
||||
async def test_greeting_request_is_removed_after_response(self):
|
||||
service = _service()
|
||||
websocket = _OpenWebSocket()
|
||||
service._websocket = websocket
|
||||
service._session_ready.set()
|
||||
service.push_frame = AsyncMock()
|
||||
|
||||
await service.speak("Welcome")
|
||||
greeting_item_id = websocket.messages[0]["item"]["id"]
|
||||
await service._handle_server_event({"type": "response.created"})
|
||||
await service._handle_server_event(
|
||||
{"type": "response.audio_transcript.done", "transcript": "Welcome"}
|
||||
)
|
||||
await service._handle_server_event(
|
||||
{"type": "response.done", "response": {"status": "completed"}}
|
||||
)
|
||||
|
||||
self.assertEqual(websocket.messages[-1]["type"], "conversation.item.delete")
|
||||
self.assertEqual(websocket.messages[-1]["item_id"], greeting_item_id)
|
||||
|
||||
def test_pipeline_sample_rates_use_qwen_defaults(self):
|
||||
cfg = AssistantConfig(
|
||||
realtime_interface_type="qwen-audio-realtime",
|
||||
realtime_values={},
|
||||
)
|
||||
self.assertEqual(realtime_audio_sample_rates(cfg), (16_000, 24_000))
|
||||
|
||||
def test_factory_and_interface_catalog_register_qwen(self):
|
||||
definition = next(
|
||||
item
|
||||
for item in INTERFACE_DEFINITIONS
|
||||
if item["interface_type"] == "qwen-audio-realtime"
|
||||
)
|
||||
self.assertEqual(definition["capability"], "Realtime")
|
||||
self.assertIn(
|
||||
"smart_turn",
|
||||
next(
|
||||
field
|
||||
for field in definition["fields"]
|
||||
if field["key"] == "turnDetection"
|
||||
)["options"],
|
||||
)
|
||||
|
||||
cfg = AssistantConfig(
|
||||
realtime_interface_type="qwen-audio-realtime",
|
||||
realtimeModel="qwen-audio-3.0-realtime-plus",
|
||||
realtime_api_key="test-key",
|
||||
realtime_base_url=(
|
||||
"wss://workspace.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime"
|
||||
),
|
||||
realtime_values={"turnDetection": "smart_turn"},
|
||||
)
|
||||
service = create_realtime_service(cfg, instructions="Be concise.")
|
||||
self.assertIsInstance(service, QwenAudioRealtimeService)
|
||||
self.assertEqual(service._turn_detection_config(), {"type": "smart_turn"})
|
||||
|
||||
async def test_resource_tester_routes_qwen_with_dashscope_header(self):
|
||||
expected = object()
|
||||
probe = AsyncMock(return_value=expected)
|
||||
with patch.object(
|
||||
model_resource_tester,
|
||||
"_test_realtime_websocket",
|
||||
probe,
|
||||
):
|
||||
result = await model_resource_tester.test_model_resource(
|
||||
"qwen-audio-realtime",
|
||||
"Realtime",
|
||||
{"apiUrl": "wss://example.test/realtime", "modelId": "model"},
|
||||
{"apiKey": "secret"},
|
||||
)
|
||||
|
||||
self.assertIs(result, expected)
|
||||
probe.assert_awaited_once_with(
|
||||
{"apiUrl": "wss://example.test/realtime", "modelId": "model"},
|
||||
{"apiKey": "secret"},
|
||||
provider="Qwen-Audio",
|
||||
extra_headers={"x-dashscope-dataInspection": "disable"},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user