wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(qwen-audio): enhance user transcript handling in QwenAudioRealtimeService

Browse Source 
- Added support for managing user transcripts, including pending states and timestamps.
- Implemented methods to handle user transcript completion and deferred assistant messages.
- Updated event handling to ensure user transcripts are emitted before assistant responses.
- Enhanced tests to verify the correct order of transcript and assistant message emissions during user interactions.
This commit is contained in:
Xin Wang
2026-07-29 17:01:02 +08:00
parent 5ef376b657
commit 510a277b5a
2 changed files with 172 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
backend/tests/test_qwen_audio_realtime.py
View File

@@ -150,6 +150,88 @@ class QwenAudioRealtimeServiceTest(unittest.IsolatedAsyncioTestCase):
self.assertEqual(end_messages[0]["content"], "Hi")
self.assertTrue(end_messages[0]["interrupted"])
async def test_user_transcript_is_emitted_before_early_assistant_delta(self):
service = _service()
service.push_frame = AsyncMock()
service.broadcast_interruption = AsyncMock()
with patch(
"services.pipecat.qwen_audio_realtime.time_now_iso8601",
side_effect=[
"2026-07-29T10:00:00+00:00",
"2026-07-29T10:00:01+00:00",
],
):
await service._handle_server_event(
{
"type": "input_audio_buffer.speech_started",
"item_id": "item_user",
}
)
await service._handle_server_event({"type": "response.created"})
await service._handle_server_event(
{
"type": "response.audio_transcript.delta",
"delta": "您好",
}
)
self.assertEqual(service.push_frame.await_count, 0)
await service._handle_server_event(
{
"type": "conversation.item.input_audio_transcription.completed",
"item_id": "item_user",
"transcript": "你好",
}
)
messages = [
call.args[0].message
for call in service.push_frame.await_args_list
if isinstance(call.args[0], OutputTransportMessageUrgentFrame)
]
self.assertEqual(
[message["type"] for message in messages],
["transcript", "assistant-text-start", "assistant-text-delta"],
)
self.assertEqual(messages[0]["timestamp"], "2026-07-29T10:00:00+00:00")
self.assertEqual(messages[1]["timestamp"], "2026-07-29T10:00:01+00:00")
async def test_transcription_failure_releases_assistant_delta(self):
service = _service()
service.push_frame = AsyncMock()
service.broadcast_interruption = AsyncMock()
await service._handle_server_event(
{
"type": "input_audio_buffer.speech_started",
"item_id": "item_user",
}
)
await service._handle_server_event({"type": "response.created"})
await service._handle_server_event(
{"type": "response.audio_transcript.delta", "delta": "您好"}
)
self.assertEqual(service.push_frame.await_count, 0)
await service._handle_server_event(
{
"type": "conversation.item.input_audio_transcription.failed",
"item_id": "item_user",
}
)
messages = [
call.args[0].message
for call in service.push_frame.await_args_list
if isinstance(call.args[0], OutputTransportMessageUrgentFrame)
]
self.assertEqual(
[message["type"] for message in messages],
["assistant-text-start", "assistant-text-delta"],
)
async def test_greeting_request_is_removed_after_response(self):
service = _service()
websocket = _OpenWebSocket()