wx44wx/pipecat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make aggregate return an AsyncIterator, other clean up

Browse Source
This commit is contained in:
Mark Backman
2025-12-03 14:46:30 -05:00
parent fa8e7458e1
commit d79dd94019
15 changed files with 259 additions and 378 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
tests/test_ivr_navigation.py
View File

@@ -10,6 +10,7 @@ from unittest.mock import AsyncMock
from pipecat.audio.vad.vad_analyzer import VADParams
from pipecat.extensions.ivr.ivr_navigator import IVRProcessor
from pipecat.frames.frames import (
LLMFullResponseEndFrame,
LLMMessagesUpdateFrame,
LLMTextFrame,
OutputDTMFUrgentFrame,
@@ -334,10 +335,12 @@ class TestIVRNavigation(unittest.IsolatedAsyncioTestCase):
frames_to_send = [
LLMTextFrame(text="Hello, I'm trying to reach billing."),
LLMFullResponseEndFrame(),
]
expected_down_frames = [
LLMTextFrame, # Should pass through unchanged
LLMFullResponseEndFrame,
]
expected_up_frames = [

68
tests/test_pattern_pair_aggregator.py
View File

@@ -38,14 +38,8 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.aggregator.on_pattern_match("code_pattern", self.code_handler)
async def test_pattern_match_and_removal(self):
# Feed text character by character
full_text = "Hello <test>pattern content</test>!"
results = []
for char in full_text:
result = await self.aggregator.aggregate(char)
if result:
results.append(result)
text = "Hello <test>pattern content</test>!"
results = [result async for result in self.aggregator.aggregate(text)]
# Verify the handler was called with correct PatternMatch object
self.test_handler.assert_called_once()
@@ -59,10 +53,8 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.assertEqual(len(results), 0)
# Next sentence should provide the lookahead and trigger the previous sentence
for char in " This is another sentence.":
result = await self.aggregator.aggregate(char)
if result:
results.append(result)
async for result in self.aggregator.aggregate(" This is another sentence."):
results.append(result)
# First result should be "Hello !" triggered by the space lookahead
self.assertEqual(len(results), 1)
@@ -77,14 +69,9 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.assertEqual(self.aggregator.text.text, "")
async def test_pattern_match_and_aggregate(self):
# Feed text character by character, collecting all results
results = []
full_text = "Here is code <code>pattern content</code> This is another sentence."
text = "Here is code <code>pattern content</code> This is another sentence."
for char in full_text:
result = await self.aggregator.aggregate(char)
if result:
results.append(result)
results = [result async for result in self.aggregator.aggregate(text)]
# First result should be "Here is code" when pattern starts
self.assertEqual(results[0].text, "Here is code")
@@ -111,15 +98,10 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.assertEqual(self.aggregator.text.text, "")
async def test_incomplete_pattern(self):
# Feed text character by character with incomplete pattern
result = None
for char in "Hello <test>pattern content":
result = await self.aggregator.aggregate(char)
if result:
break
text = "Hello <test>pattern content"
results = [result async for result in self.aggregator.aggregate(text)]
# No complete pattern yet, so nothing should be returned
self.assertIsNone(result)
self.assertEqual(len(results), 0)
# The handler should not be called yet
self.test_handler.assert_not_called()
@@ -154,13 +136,8 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.aggregator.on_pattern_match("voice", voice_handler)
self.aggregator.on_pattern_match("emphasis", emphasis_handler)
# Feed text character by character
text = "Hello <voice>female</voice> I am <em>very</em> excited to meet you!"
result = None
for char in text:
result = await self.aggregator.aggregate(char)
if result:
break
results = [result async for result in self.aggregator.aggregate(text)]
# Both handlers should be called with correct data
voice_handler.assert_called_once()
@@ -174,7 +151,7 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.assertEqual(emphasis_match.text, "very")
# With lookahead, we need to flush to get the final sentence
self.assertIsNone(result) # Waiting for lookahead after "!"
self.assertEqual(len(results), 0) # Waiting for lookahead after "!"
result = await self.aggregator.flush()
# Voice pattern should be removed, emphasis pattern should remain
@@ -184,14 +161,9 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.assertEqual(self.aggregator.text.text, "")
async def test_handle_interruption(self):
# Feed text character by character with incomplete pattern
result = None
for char in "Hello <test>pattern":
result = await self.aggregator.aggregate(char)
if result:
break
self.assertIsNone(result)
text = "Hello <test>pattern"
results = [result async for result in self.aggregator.aggregate(text)]
self.assertEqual(len(results), 0)
# Simulate interruption
await self.aggregator.handle_interruption()
@@ -203,14 +175,8 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.test_handler.assert_not_called()
async def test_pattern_across_sentences(self):
# Feed text character by character - pattern spans multiple sentences
full_text = "Hello <test>This is sentence one. This is sentence two.</test> Final sentence."
result = None
for char in full_text:
result = await self.aggregator.aggregate(char)
if result:
break
text = "Hello <test>This is sentence one. This is sentence two.</test> Final sentence."
results = [result async for result in self.aggregator.aggregate(text)]
# Handler should be called with entire content
self.test_handler.assert_called_once()
@@ -218,7 +184,7 @@ class TestPatternPairAggregator(unittest.IsolatedAsyncioTestCase):
self.assertEqual(call_args.text, "This is sentence one. This is sentence two.")
# With lookahead, we need to flush to get the final sentence
self.assertIsNone(result) # Waiting for lookahead after "."
self.assertEqual(len(results), 0) # Waiting for lookahead after "."
result = await self.aggregator.flush()
# Pattern should be removed, resulting in text with sentences merged

125
tests/test_simple_text_aggregator.py
View File

@@ -14,19 +14,22 @@ class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase):
self.aggregator = SimpleTextAggregator()
async def test_reset_aggregations(self):
# Feed character-by-character
for char in "Hello ":
assert await self.aggregator.aggregate(char) == None
text = "Hello "
results = [agg async for agg in self.aggregator.aggregate(text)]
# No complete sentences yet
assert len(results) == 0
assert self.aggregator.text.text == "Hello"
await self.aggregator.reset()
assert self.aggregator.text.text == ""
async def test_simple_sentence(self):
# Feed character-by-character: "Hello Pipecat!"
for char in "Hello Pipecat":
assert await self.aggregator.aggregate(char) == None
# After "!", lookahead waits for confirmation
assert await self.aggregator.aggregate("!") == None
text = "Hello Pipecat!"
results = [agg async for agg in self.aggregator.aggregate(text)]
# No complete sentences yet (waiting for lookahead after "!")
assert len(results) == 0
# Flush to get the pending sentence
aggregate = await self.aggregator.flush()
assert aggregate.text == "Hello Pipecat!"
@@ -34,81 +37,58 @@ class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase):
assert self.aggregator.text.text == ""
async def test_multiple_sentences(self):
# Feed character-by-character: "Hello Pipecat! How are you?"
for char in "Hello Pipecat":
assert await self.aggregator.aggregate(char) == None
# Hit "!" - wait for lookahead
assert await self.aggregator.aggregate("!") == None
# Space is whitespace - keep waiting
assert await self.aggregator.aggregate(" ") == None
# "H" confirms sentence end
result = await self.aggregator.aggregate("H")
assert result.text == "Hello Pipecat!"
# Continue with second sentence
for char in "ow are you":
assert await self.aggregator.aggregate(char) == None
# Hit "?" - wait for lookahead
assert await self.aggregator.aggregate("?") == None
text = "Hello Pipecat! How are you?"
results = [agg async for agg in self.aggregator.aggregate(text)]
# First sentence should be complete (lookahead from "H" confirmed it)
assert len(results) == 1
assert results[0].text == "Hello Pipecat!"
# Flush to get the pending sentence
result = await self.aggregator.flush()
assert result.text == "How are you?"
async def test_lookahead_decimal_number(self):
"""Test that $29.95 is not split at $29."""
# Feed character by character: "Ask me for only $29.95/month."
for char in "Ask me for only $29":
assert await self.aggregator.aggregate(char) == None
# When we hit ".", it looks like end of sentence, but should wait for lookahead
assert await self.aggregator.aggregate(".") == None
# Next character "9" confirms it's not end of sentence (NLTK changes boundary)
for char in "95/month":
assert await self.aggregator.aggregate(char) == None
# Now we hit the real end of sentence - wait for lookahead
assert await self.aggregator.aggregate(".") == None
text = "Ask me for only $29.95/month."
results = [agg async for agg in self.aggregator.aggregate(text)]
# No complete sentences yet (waiting for lookahead after final ".")
assert len(results) == 0
# Can use flush() to get the pending sentence at end of stream
result = await self.aggregator.flush()
assert result.text == "Ask me for only $29.95/month."
async def test_lookahead_abbreviation(self):
"""Test that Mr. Smith is not split at Mr."""
# Feed character by character: "Hello Mr. Smith."
for char in "Hello Mr":
assert await self.aggregator.aggregate(char) == None
# When we hit ".", it looks like end of sentence, but should wait for lookahead
assert await self.aggregator.aggregate(".") == None
# Space alone is not enough
assert await self.aggregator.aggregate(" ") == None
# "S" confirms it's not end of sentence (NLTK changes boundary detection)
for char in "Smith":
assert await self.aggregator.aggregate(char) == None
# Now we hit the real end of sentence - wait for lookahead
assert await self.aggregator.aggregate(".") == None
text = "Hello Mr. Smith."
results = [agg async for agg in self.aggregator.aggregate(text)]
# No complete sentences yet (waiting for lookahead after final ".")
assert len(results) == 0
# Can use flush() to get the pending sentence at end of stream
result = await self.aggregator.flush()
assert result.text == "Hello Mr. Smith."
async def test_lookahead_actual_sentence_end(self):
"""Test that a real sentence end is detected after lookahead."""
# Feed character by character: "Hello world. Next sentence"
for char in "Hello world":
assert await self.aggregator.aggregate(char) == None
# Hit period - should wait for lookahead
assert await self.aggregator.aggregate(".") == None
# Space alone is not enough - need non-whitespace for meaningful lookahead
assert await self.aggregator.aggregate(" ") == None
# Capital letter confirms sentence end (NLTK detects boundary at same position)
result = await self.aggregator.aggregate("N")
assert result.text == "Hello world."
# Continue with next sentence
assert await self.aggregator.aggregate("e") == None
text = "Hello world. Next sentence"
results = [agg async for agg in self.aggregator.aggregate(text)]
# First sentence should be complete (lookahead from "N" confirmed it)
assert len(results) == 1
assert results[0].text == "Hello world."
async def test_flush_pending_sentence(self):
"""Test that flush() returns pending sentence waiting for lookahead."""
# Feed up to a period
for char in "Hello world":
assert await self.aggregator.aggregate(char) == None
assert await self.aggregator.aggregate(".") == None
# At this point, "Hello world." is pending lookahead
text = "Hello world."
results = [agg async for agg in self.aggregator.aggregate(text)]
# No complete sentences yet (waiting for lookahead)
assert len(results) == 0
# Call flush to get it
result = await self.aggregator.flush()
assert result is not None
@@ -118,7 +98,12 @@ class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase):
async def test_flush_with_no_pending(self):
"""Test that flush() returns any remaining text in buffer."""
assert await self.aggregator.aggregate("Hello") == None
text = "Hello"
results = [agg async for agg in self.aggregator.aggregate(text)]
# No complete sentences
assert len(results) == 0
result = await self.aggregator.flush()
# flush() now returns any remaining text, not just pending lookahead
assert result is not None
@@ -128,13 +113,13 @@ class TestSimpleTextAggregator(unittest.IsolatedAsyncioTestCase):
async def test_flush_after_lookahead_confirmed(self):
"""Test flush after lookahead has already confirmed sentence."""
for char in "Hello.":
await self.aggregator.aggregate(char)
# Space alone is not enough - still waiting
assert await self.aggregator.aggregate(" ") == None
# Non-whitespace lookahead confirms it's a sentence
result = await self.aggregator.aggregate("W")
assert result.text == "Hello."
text = "Hello. W"
results = [agg async for agg in self.aggregator.aggregate(text)]
# First sentence should be complete (lookahead from "W" confirmed it)
assert len(results) == 1
assert results[0].text == "Hello."
# flush() returns any remaining text (the "W" in this case)
result = await self.aggregator.flush()
assert result.text == "W"

37
tests/test_skip_tags_aggregator.py
View File

@@ -17,15 +17,11 @@ class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase):
await self.aggregator.reset()
# No tags involved, aggregate at end of sentence.
# Feed text character by character
result = None
for char in "Hello Pipecat!":
result = await self.aggregator.aggregate(char)
if result:
break
text = "Hello Pipecat!"
results = [agg async for agg in self.aggregator.aggregate(text)]
# Should still be waiting for lookahead after "!"
self.assertIsNone(result)
self.assertEqual(len(results), 0)
# Flush to get the pending sentence
result = await self.aggregator.flush()
@@ -37,15 +33,11 @@ class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase):
await self.aggregator.reset()
# Tags involved, avoid aggregation during tags.
# Feed text character by character
result = None
for char in "My email is <spell>foo@pipecat.ai</spell>.":
result = await self.aggregator.aggregate(char)
if result:
break
text = "My email is <spell>foo@pipecat.ai</spell>."
results = [agg async for agg in self.aggregator.aggregate(text)]
# Should still be waiting for lookahead after "."
self.assertIsNone(result)
self.assertEqual(len(results), 0)
# Flush to get the pending sentence
result = await self.aggregator.flush()
@@ -56,22 +48,17 @@ class TestSkipTagsAggregator(unittest.IsolatedAsyncioTestCase):
async def test_streaming_tags(self):
await self.aggregator.reset()
# Tags involved, feed character by character
full_text = "My email is <spell>foo.bar@pipecat.ai</spell>."
result = None
for char in full_text:
result = await self.aggregator.aggregate(char)
if result:
break
# Tags involved
text = "My email is <spell>foo.bar@pipecat.ai</spell>."
results = [agg async for agg in self.aggregator.aggregate(text)]
# Should still be waiting for lookahead after "."
self.assertIsNone(result)
self.assertEqual(self.aggregator.text.text, full_text)
self.assertEqual(len(results), 0)
self.assertEqual(self.aggregator.text.text, text)
self.assertEqual(self.aggregator.text.type, "sentence")
# Flush to get the pending sentence
result = await self.aggregator.flush()
self.assertEqual(result.text, full_text)
self.assertEqual(result.text, text)
self.assertEqual(self.aggregator.text.text, "")
self.assertEqual(self.aggregator.text.type, "sentence")

34
tests/test_utils_string.py
View File

@@ -6,7 +6,7 @@
import unittest
from pipecat.utils.string import match_endofsentence, parse_start_end_tags, split_text_by_characters
from pipecat.utils.string import match_endofsentence, parse_start_end_tags
class TestUtilsString(unittest.IsolatedAsyncioTestCase):
@@ -232,35 +232,3 @@ class TestStartEndTags(unittest.IsolatedAsyncioTestCase):
("<a>", "</a>"),
41,
)
async def test_split_text_by_characters(self):
"""Test splitting text into individual characters."""
# Basic sentence
assert split_text_by_characters("Hello world!") == [
"H",
"e",
"l",
"l",
"o",
" ",
"w",
"o",
"r",
"l",
"d",
"!",
]
# Single word
assert split_text_by_characters("Hi") == ["H", "i"]
# Empty string
assert split_text_by_characters("") == []
# With spaces
assert split_text_by_characters("A B") == ["A", " ", "B"]
# Concatenation test - characters should concatenate back to original
characters = split_text_by_characters("Hello world!")
concatenated = "".join(characters)
assert concatenated == "Hello world!"