774825593d
- Introduced Qwen-Audio Realtime service for speech-to-speech processing in Pipecat. - Updated interface catalog to include Qwen-Audio Realtime capabilities. - Enhanced model resource testing to support new service. - Added configuration options for audio sample rates and turn detection modes. - Updated documentation to reflect integration details and usage instructions.
235 lines
8.6 KiB
Python
235 lines
8.6 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import json
|
|
import unittest
|
|
from unittest.mock import AsyncMock, patch
|
|
|
|
from models import AssistantConfig
|
|
from pipecat.frames.frames import OutputTransportMessageUrgentFrame, TTSAudioRawFrame
|
|
from services import model_resource_tester
|
|
from services.interface_catalog import INTERFACE_DEFINITIONS
|
|
from services.pipecat.qwen_audio_realtime import QwenAudioRealtimeService
|
|
from services.pipecat.service_factory import (
|
|
create_realtime_service,
|
|
realtime_audio_sample_rates,
|
|
)
|
|
from websockets.protocol import State
|
|
|
|
|
|
class _OpenWebSocket:
|
|
state = State.OPEN
|
|
|
|
def __init__(self) -> None:
|
|
self.messages: list[dict] = []
|
|
|
|
async def send(self, raw_message: str) -> None:
|
|
self.messages.append(json.loads(raw_message))
|
|
|
|
|
|
def _service(**overrides) -> QwenAudioRealtimeService:
|
|
settings = {
|
|
"api_key": "test-key",
|
|
"model": "qwen-audio-3.0-realtime-flash",
|
|
"base_url": (
|
|
"wss://workspace.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime"
|
|
),
|
|
"instructions": "You are helpful.",
|
|
}
|
|
settings.update(overrides)
|
|
return QwenAudioRealtimeService(**settings)
|
|
|
|
|
|
class QwenAudioRealtimeServiceTest(unittest.IsolatedAsyncioTestCase):
|
|
def test_provider_defaults_and_session_config(self):
|
|
service = _service()
|
|
|
|
self.assertEqual(
|
|
service._connection_url(),
|
|
"wss://workspace.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime"
|
|
"?model=qwen-audio-3.0-realtime-flash",
|
|
)
|
|
self.assertEqual(
|
|
service._initial_session_config(),
|
|
{
|
|
"modalities": ["text", "audio"],
|
|
"instructions": "You are helpful.",
|
|
"voice": "longanqian",
|
|
"input_audio_format": "pcm",
|
|
"output_audio_format": "pcm",
|
|
"turn_detection": {
|
|
"type": "server_vad",
|
|
"threshold": 0.5,
|
|
"silence_duration_ms": 800,
|
|
},
|
|
"max_history_turns": 20,
|
|
},
|
|
)
|
|
|
|
def test_smart_turn_and_value_ranges_are_explicit(self):
|
|
service = _service(turn_detection_mode="smart_turn")
|
|
self.assertEqual(service._turn_detection_config(), {"type": "smart_turn"})
|
|
|
|
with self.assertRaisesRegex(ValueError, "turn detection mode"):
|
|
_service(turn_detection_mode="manual")
|
|
with self.assertRaisesRegex(ValueError, "threshold"):
|
|
_service(vad_threshold=1.1)
|
|
with self.assertRaisesRegex(ValueError, "history"):
|
|
_service(max_history_turns=51)
|
|
|
|
async def test_session_ready_flushes_queued_events_in_order(self):
|
|
service = _service()
|
|
websocket = _OpenWebSocket()
|
|
service._websocket = websocket
|
|
|
|
await service.send_text("hello")
|
|
self.assertEqual(websocket.messages, [])
|
|
|
|
await service._handle_server_event({"type": "session.created"})
|
|
self.assertEqual(websocket.messages[0]["type"], "session.update")
|
|
self.assertEqual(
|
|
websocket.messages[0]["session"]["input_audio_format"], "pcm"
|
|
)
|
|
|
|
await service._handle_server_event({"type": "session.updated"})
|
|
self.assertEqual(
|
|
[message["type"] for message in websocket.messages],
|
|
["session.update", "conversation.item.create", "response.create"],
|
|
)
|
|
|
|
async def test_dynamic_instructions_do_not_resend_locked_settings(self):
|
|
service = _service()
|
|
websocket = _OpenWebSocket()
|
|
service._websocket = websocket
|
|
service._session_ready.set()
|
|
|
|
await service.update_instructions("Call the user Alice.")
|
|
|
|
self.assertEqual(websocket.messages[-1]["type"], "session.update")
|
|
self.assertEqual(
|
|
websocket.messages[-1]["session"],
|
|
{"instructions": "Call the user Alice."},
|
|
)
|
|
|
|
async def test_interruption_cancels_and_suppresses_residual_audio(self):
|
|
service = _service()
|
|
websocket = _OpenWebSocket()
|
|
service._websocket = websocket
|
|
service._session_ready.set()
|
|
service.push_frame = AsyncMock()
|
|
service.broadcast_interruption = AsyncMock()
|
|
audio = base64.b64encode(b"\x01\x02").decode("ascii")
|
|
|
|
await service._handle_server_event({"type": "response.created"})
|
|
await service._handle_server_event(
|
|
{"type": "response.audio_transcript.delta", "delta": "Hi"}
|
|
)
|
|
await service._handle_server_event(
|
|
{"type": "response.audio.delta", "delta": audio}
|
|
)
|
|
await service._handle_server_event(
|
|
{"type": "input_audio_buffer.speech_started"}
|
|
)
|
|
calls_before_residual_audio = service.push_frame.await_count
|
|
await service._handle_server_event(
|
|
{"type": "response.audio.delta", "delta": audio}
|
|
)
|
|
|
|
self.assertEqual(service.push_frame.await_count, calls_before_residual_audio)
|
|
self.assertIn("response.cancel", [item["type"] for item in websocket.messages])
|
|
service.broadcast_interruption.assert_awaited_once()
|
|
|
|
frames = [call.args[0] for call in service.push_frame.await_args_list]
|
|
self.assertTrue(any(isinstance(frame, TTSAudioRawFrame) for frame in frames))
|
|
end_messages = [
|
|
frame.message
|
|
for frame in frames
|
|
if isinstance(frame, OutputTransportMessageUrgentFrame)
|
|
and frame.message.get("type") == "assistant-text-end"
|
|
]
|
|
self.assertEqual(end_messages[0]["content"], "Hi")
|
|
self.assertTrue(end_messages[0]["interrupted"])
|
|
|
|
async def test_greeting_request_is_removed_after_response(self):
|
|
service = _service()
|
|
websocket = _OpenWebSocket()
|
|
service._websocket = websocket
|
|
service._session_ready.set()
|
|
service.push_frame = AsyncMock()
|
|
|
|
await service.speak("Welcome")
|
|
greeting_item_id = websocket.messages[0]["item"]["id"]
|
|
await service._handle_server_event({"type": "response.created"})
|
|
await service._handle_server_event(
|
|
{"type": "response.audio_transcript.done", "transcript": "Welcome"}
|
|
)
|
|
await service._handle_server_event(
|
|
{"type": "response.done", "response": {"status": "completed"}}
|
|
)
|
|
|
|
self.assertEqual(websocket.messages[-1]["type"], "conversation.item.delete")
|
|
self.assertEqual(websocket.messages[-1]["item_id"], greeting_item_id)
|
|
|
|
def test_pipeline_sample_rates_use_qwen_defaults(self):
|
|
cfg = AssistantConfig(
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
realtime_values={},
|
|
)
|
|
self.assertEqual(realtime_audio_sample_rates(cfg), (16_000, 24_000))
|
|
|
|
def test_factory_and_interface_catalog_register_qwen(self):
|
|
definition = next(
|
|
item
|
|
for item in INTERFACE_DEFINITIONS
|
|
if item["interface_type"] == "qwen-audio-realtime"
|
|
)
|
|
self.assertEqual(definition["capability"], "Realtime")
|
|
self.assertIn(
|
|
"smart_turn",
|
|
next(
|
|
field
|
|
for field in definition["fields"]
|
|
if field["key"] == "turnDetection"
|
|
)["options"],
|
|
)
|
|
|
|
cfg = AssistantConfig(
|
|
realtime_interface_type="qwen-audio-realtime",
|
|
realtimeModel="qwen-audio-3.0-realtime-plus",
|
|
realtime_api_key="test-key",
|
|
realtime_base_url=(
|
|
"wss://workspace.cn-beijing.maas.aliyuncs.com/api-ws/v1/realtime"
|
|
),
|
|
realtime_values={"turnDetection": "smart_turn"},
|
|
)
|
|
service = create_realtime_service(cfg, instructions="Be concise.")
|
|
self.assertIsInstance(service, QwenAudioRealtimeService)
|
|
self.assertEqual(service._turn_detection_config(), {"type": "smart_turn"})
|
|
|
|
async def test_resource_tester_routes_qwen_with_dashscope_header(self):
|
|
expected = object()
|
|
probe = AsyncMock(return_value=expected)
|
|
with patch.object(
|
|
model_resource_tester,
|
|
"_test_realtime_websocket",
|
|
probe,
|
|
):
|
|
result = await model_resource_tester.test_model_resource(
|
|
"qwen-audio-realtime",
|
|
"Realtime",
|
|
{"apiUrl": "wss://example.test/realtime", "modelId": "model"},
|
|
{"apiKey": "secret"},
|
|
)
|
|
|
|
self.assertIs(result, expected)
|
|
probe.assert_awaited_once_with(
|
|
{"apiUrl": "wss://example.test/realtime", "modelId": "model"},
|
|
{"apiKey": "secret"},
|
|
provider="Qwen-Audio",
|
|
extra_headers={"x-dashscope-dataInspection": "disable"},
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|