wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'main' into filipi/includes_inter_frame_spaces

Browse Source 
# Conflicts:
#	uv.lock
This commit is contained in:
filipi87
2026-04-22 15:22:06 -03:00
parent bba7ca80e3 3f3d3c9203
commit ac810e57ed
374 changed files with 3688 additions and 1372 deletions
Download Patch File Download Diff File Expand all files Collapse all files

236
tests/test_krisp_ip_user_turn_start_strategy.py Normal file
View File

@@ -0,0 +1,236 @@
#
# Copyright (c) 2024-2026, Daily
#
# SPDX-License-Identifier: BSD 2-Clause License
#
import os
import sys
import tempfile
import unittest
from unittest.mock import MagicMock, patch
import numpy as np
# Mock package version check before importing pipecat (development mode)
_version_patcher = patch("importlib.metadata.version", return_value="0.0.0-dev")
_version_patcher.start()
# Mock krisp_audio before any pipecat import that loads krisp_instance / VIVA IP strategy
mock_krisp_audio = MagicMock()
mock_krisp_audio.SamplingRate.Sr8000Hz = 8000
mock_krisp_audio.SamplingRate.Sr16000Hz = 16000
mock_krisp_audio.SamplingRate.Sr24000Hz = 24000
mock_krisp_audio.SamplingRate.Sr32000Hz = 32000
mock_krisp_audio.SamplingRate.Sr44100Hz = 44100
mock_krisp_audio.SamplingRate.Sr48000Hz = 48000
mock_krisp_audio.FrameDuration.Fd10ms = "10ms"
mock_krisp_audio.FrameDuration.Fd15ms = "15ms"
mock_krisp_audio.FrameDuration.Fd20ms = "20ms"
mock_krisp_audio.FrameDuration.Fd30ms = "30ms"
mock_krisp_audio.FrameDuration.Fd32ms = "32ms"
sys.modules["krisp_audio"] = mock_krisp_audio
mock_pipecat_krisp = MagicMock()
sys.modules["pipecat_ai_krisp"] = mock_pipecat_krisp
sys.modules["pipecat_ai_krisp.audio"] = MagicMock()
sys.modules["pipecat_ai_krisp.audio.krisp_processor"] = MagicMock()
from pipecat.frames.frames import (
BotStartedSpeakingFrame,
BotStoppedSpeakingFrame,
InputAudioRawFrame,
TranscriptionFrame,
VADUserStartedSpeakingFrame,
VADUserStoppedSpeakingFrame,
)
from pipecat.turns.types import ProcessFrameResult
from pipecat.turns.user_start.krisp_viva_ip_user_turn_start_strategy import (
KrispVivaIPUserTurnStartStrategy,
)
STRATEGY_MODULE = "pipecat.turns.user_start.krisp_viva_ip_user_turn_start_strategy"
def _int16_silence(num_samples: int) -> bytes:
return np.zeros(num_samples, dtype=np.int16).tobytes()
class TestKrispVivaIPUserTurnStartStrategy(unittest.IsolatedAsyncioTestCase):
"""Tests for KrispVivaIPUserTurnStartStrategy with mocked krisp_audio."""
def setUp(self):
self.temp_model_file = tempfile.NamedTemporaryFile(suffix=".kef", delete=False)
self.temp_model_file.write(b"dummy")
self.temp_model_file.close()
self.model_path = self.temp_model_file.name
self.mock_krisp_audio = mock_krisp_audio
self.mock_krisp_audio.reset_mock()
self.mock_krisp_audio.ModelInfo.reset_mock()
self.mock_krisp_audio.IpSessionConfig.reset_mock()
self.mock_krisp_audio.IpFloat.reset_mock()
self.mock_model_info = MagicMock()
self.mock_krisp_audio.ModelInfo.return_value = self.mock_model_info
self.mock_ip_cfg = MagicMock()
self.mock_krisp_audio.IpSessionConfig.return_value = self.mock_ip_cfg
self.mock_ip_session = MagicMock()
self.mock_krisp_audio.IpFloat.create.return_value = self.mock_ip_session
self.krisp_patch = patch(f"{STRATEGY_MODULE}.krisp_audio", self.mock_krisp_audio)
self.krisp_patch.start()
self.sdk_patcher = patch(f"{STRATEGY_MODULE}.KrispVivaSDKManager")
self.mock_sdk_manager = self.sdk_patcher.start()
self.mock_sdk_manager.acquire = MagicMock()
self.mock_sdk_manager.release = MagicMock()
def tearDown(self):
self.krisp_patch.stop()
self.sdk_patcher.stop()
if os.path.exists(self.model_path):
os.unlink(self.model_path)
def _make_strategy(self, *, threshold: float = 0.5, frame_duration_ms: int = 20):
return KrispVivaIPUserTurnStartStrategy(
model_path=self.model_path,
threshold=threshold,
frame_duration_ms=frame_duration_ms,
api_key="test-key",
)
def _audio_frame(self, sample_rate: int = 16000, frame_duration_ms: int = 20):
samples = int(sample_rate * frame_duration_ms / 1000)
return InputAudioRawFrame(
audio=_int16_silence(samples),
sample_rate=sample_rate,
num_channels=1,
)
async def test_interruption_detected_emits_turn_and_stop(self):
self.mock_ip_session.process = MagicMock(return_value=0.87)
strategy = self._make_strategy(threshold=0.5)
try:
fired = False
@strategy.event_handler("on_user_turn_started")
async def on_user_turn_started(strategy, params):
nonlocal fired
fired = True
await strategy.process_frame(VADUserStartedSpeakingFrame())
result = await strategy.process_frame(self._audio_frame())
self.assertTrue(fired)
self.assertEqual(result, ProcessFrameResult.STOP)
self.mock_ip_session.process.assert_called()
finally:
await strategy.cleanup()
async def test_backchannel_suppressed_no_event_continue(self):
self.mock_ip_session.process = MagicMock(return_value=0.23)
strategy = self._make_strategy(threshold=0.5)
try:
fired = False
@strategy.event_handler("on_user_turn_started")
async def on_user_turn_started(strategy, params):
nonlocal fired
fired = True
await strategy.process_frame(VADUserStartedSpeakingFrame())
result = await strategy.process_frame(self._audio_frame())
self.assertFalse(fired)
self.assertEqual(result, ProcessFrameResult.CONTINUE)
finally:
await strategy.cleanup()
async def test_reset_on_vad_stopped_clears_state(self):
self.mock_ip_session.process = MagicMock(return_value=0.1)
strategy = self._make_strategy(threshold=0.5)
try:
await strategy.process_frame(VADUserStartedSpeakingFrame())
await strategy.process_frame(self._audio_frame())
self.mock_ip_session.process.reset_mock()
await strategy.process_frame(VADUserStoppedSpeakingFrame())
result = await strategy.process_frame(self._audio_frame())
self.assertEqual(result, ProcessFrameResult.CONTINUE)
self.mock_ip_session.process.assert_not_called()
finally:
await strategy.cleanup()
async def test_reset_on_bot_stopped_clears_state(self):
self.mock_ip_session.process = MagicMock(return_value=0.1)
strategy = self._make_strategy(threshold=0.5)
try:
await strategy.process_frame(VADUserStartedSpeakingFrame())
await strategy.process_frame(self._audio_frame())
self.mock_ip_session.process.reset_mock()
await strategy.process_frame(BotStoppedSpeakingFrame())
result = await strategy.process_frame(self._audio_frame())
self.assertEqual(result, ProcessFrameResult.CONTINUE)
self.mock_ip_session.process.assert_not_called()
finally:
await strategy.cleanup()
async def test_no_op_before_vad_start(self):
self.mock_ip_session.process = MagicMock(return_value=0.99)
strategy = self._make_strategy()
try:
result = await strategy.process_frame(self._audio_frame())
self.assertEqual(result, ProcessFrameResult.CONTINUE)
self.mock_ip_session.process.assert_not_called()
finally:
await strategy.cleanup()
async def test_decision_sticks_no_double_trigger(self):
self.mock_ip_session.process = MagicMock(return_value=0.9)
strategy = self._make_strategy(threshold=0.5)
try:
count = 0
@strategy.event_handler("on_user_turn_started")
async def on_user_turn_started(strategy, params):
nonlocal count
count += 1
await strategy.process_frame(VADUserStartedSpeakingFrame())
r1 = await strategy.process_frame(self._audio_frame())
r2 = await strategy.process_frame(self._audio_frame())
self.assertEqual(r1, ProcessFrameResult.STOP)
self.assertEqual(r2, ProcessFrameResult.CONTINUE)
self.assertEqual(count, 1)
finally:
await strategy.cleanup()
async def test_unrelated_frames_continue(self):
strategy = self._make_strategy()
try:
r1 = await strategy.process_frame(BotStartedSpeakingFrame())
r2 = await strategy.process_frame(
TranscriptionFrame(text="hi", user_id="", timestamp="")
)
self.assertEqual(r1, ProcessFrameResult.CONTINUE)
self.assertEqual(r2, ProcessFrameResult.CONTINUE)
finally:
await strategy.cleanup()
if __name__ == "__main__":
unittest.main()

7
tests/test_krisp_sdk_manager.py
View File

@@ -62,8 +62,15 @@ class TestKrispVivaSDKManager:
def setup_method(self):
"""Reset mocks and SDK state before each test."""
mock_krisp_audio.reset_mock()
mock_krisp_audio.globalInit.side_effect = None
mock_krisp_audio.getVersion.return_value = mock_version
# Ensure krisp_instance module uses THIS test's mock, not a stale
# reference cached from a different test file's sys.modules entry.
import pipecat.audio.krisp_instance as _ki
_ki.krisp_audio = mock_krisp_audio
# Reset the SDK manager state for clean tests
# We access internal state to ensure tests are isolated
with KrispVivaSDKManager._lock:

101
tests/test_user_turn_stop_strategy.py
View File

@@ -529,6 +529,107 @@ class TestSpeechTimeoutUserTurnStopStrategy(unittest.IsolatedAsyncioTestCase):
# Non-finalized transcript received after timeout, triggers immediately
self.assertTrue(should_start)
async def test_finalized_short_circuits_stt_wait(self):
"""Finalized transcript cancels the stt_timeout safety net.
user_speech_timeout still runs to completion as a policy floor,
but stt_timeout is skipped once STT says it's done. Net effect:
the turn stops at user_speech_timeout, not stt_timeout.
"""
stt_timeout = AGGREGATION_TIMEOUT * 4
strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
await strategy.process_frame(
STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout)
)
should_start = None
@strategy.event_handler("on_user_turn_stopped")
async def on_user_turn_stopped(strategy, params):
nonlocal should_start
should_start = True
# S → E: starts user_speech_timeout (short) and stt_timeout (long).
await strategy.process_frame(VADUserStartedSpeakingFrame())
await strategy.process_frame(VADUserStoppedSpeakingFrame())
# Finalized transcript arrives before user_speech_timeout elapses.
await strategy.process_frame(
TranscriptionFrame(text="Hello!", user_id="cat", timestamp="", finalized=True)
)
# user_speech_timeout is still running, so no trigger yet.
self.assertIsNone(should_start)
# user_speech_timeout elapses — stt_timeout was short-circuited,
# so the turn stops now rather than waiting for stt_timeout.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_non_finalized_waits_full_stt_timeout(self):
"""Non-finalized transcript does not short-circuit stt_timeout.
When STT never signals finalization, the stt_timeout safety net
must run its full course — the turn should not stop until the
longer of the two timers has elapsed.
"""
stt_timeout = AGGREGATION_TIMEOUT * 4
strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
await strategy.process_frame(
STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout)
)
should_start = None
@strategy.event_handler("on_user_turn_stopped")
async def on_user_turn_stopped(strategy, params):
nonlocal should_start
should_start = True
# S → E: both timers start.
await strategy.process_frame(VADUserStartedSpeakingFrame())
await strategy.process_frame(VADUserStoppedSpeakingFrame())
# Non-finalized transcript during the wait.
await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp=""))
# user_speech_timeout elapses but stt_timeout has not — no trigger.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertIsNone(should_start)
# Wait for the remainder of stt_timeout.
await asyncio.sleep(stt_timeout - AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_fallback_uses_only_user_speech_timeout(self):
"""Fallback path (no VAD) ignores stt_timeout and uses only user_speech_timeout.
stt_timeout is defined as "p99 after VAD stop" — without a VAD
reference point it has no meaning. The fallback measures
inactivity since the last transcript, which is user_speech_timeout.
"""
stt_timeout = AGGREGATION_TIMEOUT * 4
strategy = SpeechTimeoutUserTurnStopStrategy(user_speech_timeout=AGGREGATION_TIMEOUT)
await strategy.setup(self.task_manager)
await strategy.process_frame(
STTMetadataFrame(service_name="test", ttfs_p99_latency=stt_timeout)
)
should_start = None
@strategy.event_handler("on_user_turn_stopped")
async def on_user_turn_stopped(strategy, params):
nonlocal should_start
should_start = True
# Transcript arrives without any VAD frame — fallback path.
await strategy.process_frame(TranscriptionFrame(text="Hello!", user_id="cat", timestamp=""))
# The fallback timer is user_speech_timeout, not stt_timeout.
await asyncio.sleep(AGGREGATION_TIMEOUT + 0.1)
self.assertTrue(should_start)
async def test_reset_clears_stale_text_no_premature_stop(self):
"""Test that reset() clears stale text and cancels timeout, preventing premature stop.

91
tests/test_xai_tts.py
View File

@@ -4,14 +4,19 @@
# SPDX-License-Identifier: BSD 2-Clause License
#
"""Tests for XAIHttpTTSService."""
"""Tests for XAIHttpTTSService and XAITTSService."""
import asyncio
import base64
import json
import unittest
from urllib.parse import parse_qs, urlparse
import aiohttp
import pytest
import websockets
from aiohttp import web
from websockets.asyncio.server import serve
from pipecat.frames.frames import (
AggregatedTextFrame,
@@ -21,7 +26,7 @@ from pipecat.frames.frames import (
TTSStoppedFrame,
TTSTextFrame,
)
from pipecat.services.xai.tts import XAIHttpTTSService
from pipecat.services.xai.tts import XAIHttpTTSService, XAITTSService
from pipecat.tests.utils import run_test
@@ -87,5 +92,87 @@ async def test_run_xai_tts_success(aiohttp_client):
}
@pytest.mark.asyncio
async def test_run_xai_websocket_tts_success():
"""xAI WS TTS should send text.delta+text.done and emit frames from audio.delta+audio.done."""
captured: dict = {
"request_path": None,
"auth_header": None,
"messages": [],
}
audio_bytes = b"\x00\x01\x02\x03" * 1024
async def handler(ws):
request = ws.request
captured["request_path"] = request.path
captured["auth_header"] = request.headers.get("Authorization")
try:
async for raw in ws:
msg = json.loads(raw)
captured["messages"].append(msg)
if msg.get("type") == "text.done":
await ws.send(
json.dumps(
{
"type": "audio.delta",
"delta": base64.b64encode(audio_bytes).decode("ascii"),
}
)
)
await ws.send(json.dumps({"type": "audio.done", "trace_id": "test-trace"}))
except websockets.ConnectionClosed:
pass
async with serve(handler, "127.0.0.1", 0) as server:
host, port = next(iter(server.sockets)).getsockname()[:2]
base_url = f"ws://{host}:{port}/v1/tts"
tts_service = XAITTSService(
api_key="test-key",
base_url=base_url,
sample_rate=24000,
)
down_frames, _ = await run_test(
tts_service,
frames_to_send=[TTSSpeakFrame(text="Hello from xAI."), _SleepAfterSpeak(0.3)],
)
frame_types = [type(frame) for frame in down_frames]
assert TTSStartedFrame in frame_types
assert TTSAudioRawFrame in frame_types
assert TTSStoppedFrame in frame_types
audio_frames = [frame for frame in down_frames if isinstance(frame, TTSAudioRawFrame)]
assert audio_frames
assert all(frame.sample_rate == 24000 for frame in audio_frames)
assert all(frame.num_channels == 1 for frame in audio_frames)
assert b"".join(f.audio for f in audio_frames) == audio_bytes
assert captured["auth_header"] == "Bearer test-key"
parsed = urlparse(captured["request_path"])
query = parse_qs(parsed.query)
assert query["voice"] == ["eve"]
assert query["language"] == ["en"]
assert query["codec"] == ["pcm"]
assert query["sample_rate"] == ["24000"]
types_sent = [m.get("type") for m in captured["messages"]]
assert "text.delta" in types_sent
assert "text.done" in types_sent
delta_msg = next(m for m in captured["messages"] if m.get("type") == "text.delta")
assert delta_msg["delta"] == "Hello from xAI."
# Small helper imported lazily to avoid circular import in fixture-lite tests.
def _SleepAfterSpeak(duration: float):
from pipecat.tests.utils import SleepFrame
return SleepFrame(sleep=duration)
if __name__ == "__main__":
unittest.main()