From 26c937af87eaed956a61180c8756cd0c09a6fcd0 Mon Sep 17 00:00:00 2001 From: Mark Backman Date: Mon, 21 Jul 2025 23:04:52 -0400 Subject: [PATCH] Update match_endofsentence to use NLTK sentence tokenizer --- CHANGELOG.md | 3 + pyproject.toml | 1 + src/pipecat/utils/string.py | 120 +++++++++++++++++++++-------- tests/test_utils_string.py | 150 ++++++++++++++++++++++++++++++++---- 4 files changed, 228 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a0b6966d..d272c0fe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated `MiniMaxHttpTTSService` with a `base_url` arg where you can specify the Global endpoint (default) or Mainland China. +- Replaced regex-based sentence detection in `match_endofsentence` with NLTK's + punkt_tab tokenizer for more reliable sentence boundary detection. + - Changed the `livekit` optional dependency for `tenacity` to `tenacity>=8.2.3,<10.0.0` in order to support the `google-genai` package. diff --git a/pyproject.toml b/pyproject.toml index c1f0f892b..b493f02e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "docstring_parser~=0.16", "loguru~=0.7.3", "Markdown~=3.7", + "nltk>=3.9.1", "numpy>=1.26.4", "Pillow~=11.1.0", "protobuf~=5.29.3", diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 21449a3ab..592d7c2a5 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -9,29 +9,72 @@ This module provides utilities for natural language text processing including sentence boundary detection, email and number pattern handling, and XML-style tag parsing for structured text content. + +Dependencies: + This module uses NLTK (Natural Language Toolkit) for robust sentence + tokenization. NLTK is licensed under the Apache License 2.0. + See: https://www.nltk.org/ + Source: https://www.nltk.org/api/nltk.tokenize.punkt.html """ import re -from typing import Optional, Sequence, Tuple +from typing import FrozenSet, Optional, Sequence, Tuple -ENDOFSENTENCE_PATTERN_STR = r""" - (? str: def match_endofsentence(text: str) -> int: """Find the position of the end of a sentence in the provided text. - This function processes the input text by replacing periods in email - addresses and numbers with ampersands to prevent them from being - misidentified as sentence terminals. It then searches for the end of a - sentence using a specified regex pattern. + This function uses NLTK's sentence tokenizer to detect sentence boundaries + in the input text, combined with punctuation verification to ensure that + single tokens without proper sentence endings aren't considered complete sentences. Args: text: The input text in which to find the end of the sentence. @@ -71,21 +113,33 @@ def match_endofsentence(text: str) -> int: """ text = text.rstrip() - # Replace email dots by ampersands so we can find the end of sentence. For - # example, first.last@email.com becomes first&last@email&com. - emails = list(EMAIL_PATTERN.finditer(text)) - for email_match in emails: - text = replace_match(text, email_match, ".", "&") + if not text: + return 0 - # Replace number dots by ampersands so we can find the end of sentence. - numbers = list(NUMBER_PATTERN.finditer(text)) - for number_match in numbers: - text = replace_match(text, number_match, ".", "&") + # Use NLTK's sentence tokenizer to find sentence boundaries + sentences = sent_tokenize(text) - # Match against the new text. - match = ENDOFSENTENCE_PATTERN.search(text) + if not sentences: + return 0 - return match.end() if match else 0 + first_sentence = sentences[0] + + # If there's only one sentence that equals the entire text, + # verify it actually ends with sentence-ending punctuation. + # This is required as NLTK may return a single sentence for + # text that's a single word. In the case of LLM tokens, it's + # common for text to be single words, so we need to ensure + # sentence-ending punctuation is present. + if len(sentences) == 1 and first_sentence == text: + return len(text) if text and text[-1] in SENTENCE_ENDING_PUNCTUATION else 0 + + # If there are multiple sentences, the first one is complete by definition + # (NLTK found a boundary, so there must be proper punctuation) + if len(sentences) > 1: + return len(first_sentence) + + # Single sentence that doesn't equal the full text means incomplete + return 0 def parse_start_end_tags( diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index cabd88a36..607b8d0fa 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -16,10 +16,13 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("This is a sentence?") == 19 assert match_endofsentence("This is a sentence;") == 19 assert match_endofsentence("This is a sentence...") == 21 - assert match_endofsentence("This is a sentence . . .") == 24 - assert match_endofsentence("This is a sentence. ..") == 22 + assert match_endofsentence("This is a sentence. This is another one") == 19 assert match_endofsentence("This is for Mr. and Mrs. Jones.") == 31 - assert match_endofsentence("U.S.A and U.S.A..") == 17 + assert match_endofsentence("Meet the new Mr. and Mrs.") == 25 + assert match_endofsentence("U.S.A. and N.A.S.A.") == 19 + assert match_endofsentence("USA and NASA.") == 13 + assert match_endofsentence("My number is 123-456-7890.") == 26 + assert match_endofsentence("For information, call 411.") == 26 assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 31 assert match_endofsentence("My email is spell(foo.bar@pipecat.ai).") == 38 @@ -27,41 +30,162 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("The number pi is 3.14159.") == 25 assert match_endofsentence("Valid scientific notation 1.23e4.") == 33 assert match_endofsentence("Valid scientific notation 0.e4.") == 31 + assert match_endofsentence("It still early, it's 3:00 a.m.") == 30 assert not match_endofsentence("This is not a sentence") assert not match_endofsentence("This is not a sentence,") assert not match_endofsentence("This is not a sentence, ") assert not match_endofsentence("Ok, Mr. Smith let's ") assert not match_endofsentence("Dr. Walker, I presume ") assert not match_endofsentence("Prof. Walker, I presume ") - assert not match_endofsentence("zweitens, und 3.") - assert not match_endofsentence("Heute ist Dienstag, der 3.") # 3. Juli 2024 - assert not match_endofsentence("America, or the U.") # U.S.A. - assert not match_endofsentence("It still early, it's 3:00 a.") # 3:00 a.m. + assert not match_endofsentence("zweitens, und 3") + assert not match_endofsentence("Heute ist Dienstag, der 3") # 3. Juli 2024 + assert not match_endofsentence("America, or the U.S") # U.S.A. assert not match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai") assert not match_endofsentence("The number pi is 3.14159") - async def test_endofsentence_zh(self): + async def test_endofsentence_multilingual(self): + """Test sentence detection across various language families and scripts.""" + + # Arabic script (Arabic, Urdu, Persian) + arabic_sentences = [ + "مرحبا؟", # Arabic question mark + "السلام عليكم؛", # Arabic semicolon + "یہ اردو ہے۔", # Urdu full stop + ] + for sentence in arabic_sentences: + assert match_endofsentence(sentence), f"Failed for Arabic/Urdu: {sentence}" + + # Should not match incomplete Arabic + assert not match_endofsentence("مرحبا،"), "Arabic comma should not end sentence" + chinese_sentences = [ "你好。", "你好！", "吃了吗？", "安全第一；", ] - for i in chinese_sentences: - assert match_endofsentence(i) + for sentence in chinese_sentences: + assert match_endofsentence(sentence), f"Failed for Chinese: {sentence}" assert not match_endofsentence("你好，") - async def test_endofsentence_hi(self): hindi_sentences = [ "हैलो।", "हैलो！", "आप खाये हैं？", "सुरक्षा पहले।", ] - for i in hindi_sentences: - assert match_endofsentence(i) + for sentence in hindi_sentences: + assert match_endofsentence(sentence), f"Failed for Hindi: {sentence}" assert not match_endofsentence("हैलो，") + # East Asian (Japanese, Korean) + japanese_sentences = [ + "こんにちは。", # Japanese + "元気ですか？", # Japanese question + "ありがとう！", # Japanese exclamation + ] + for sentence in japanese_sentences: + assert match_endofsentence(sentence), f"Failed for Japanese: {sentence}" + + korean_sentences = [ + "안녕하세요。", # Korean with ideographic period + "어떻게 지내세요？", # Korean question + ] + for sentence in korean_sentences: + assert match_endofsentence(sentence), f"Failed for Korean: {sentence}" + + # Southeast Asian scripts + thai_sentences = [ + "สวัสดี।", # Thai with Devanagari-style punctuation + ] + for sentence in thai_sentences: + assert match_endofsentence(sentence), f"Failed for Thai: {sentence}" + + myanmar_sentences = [ + "မင်္ဂလာပါ၊", # Myanmar little section + "ကျေးဇူးတင်ပါတယ်။", # Myanmar section + ] + for sentence in myanmar_sentences: + assert match_endofsentence(sentence), f"Failed for Myanmar: {sentence}" + + # Other Indic scripts (same punctuation as Hindi but different scripts) + bengali_sentences = [ + "নমস্কার।", # Bengali + "আপনি কেমন আছেন？", # Bengali question (uses Latin ?) + ] + for sentence in bengali_sentences: + assert match_endofsentence(sentence), f"Failed for Bengali: {sentence}" + + tamil_sentences = [ + "வணக்கம்।", # Tamil + "நீங்கள் எப்படி இருக்கிறீர்கள்？", # Tamil question + ] + for sentence in tamil_sentences: + assert match_endofsentence(sentence), f"Failed for Tamil: {sentence}" + + # Armenian + armenian_sentences = [ + "Բարև։", # Armenian full stop + "Ինչպես եք՞", # Armenian question mark + "Շնորհակալություն՜", # Armenian exclamation + ] + for sentence in armenian_sentences: + assert match_endofsentence(sentence), f"Failed for Armenian: {sentence}" + + # Ethiopic (Amharic) + amharic_sentences = [ + "ሰላም።", # Ethiopic full stop + "እንዴት ነዎት፧", # Ethiopic question mark + ] + for sentence in amharic_sentences: + assert match_endofsentence(sentence), f"Failed for Amharic: {sentence}" + + # Languages using Latin punctuation (should still work) + latin_script_sentences = [ + "Hola.", # Spanish + "Bonjour!", # French + "Guten Tag?", # German + "Привет.", # Russian (Cyrillic but uses Latin punctuation) + "Γεια σας.", # Greek + "שלום.", # Hebrew + "გამარჯობა.", # Georgian + ] + for sentence in latin_script_sentences: + assert match_endofsentence(sentence), f"Failed for Latin script: {sentence}" + + async def test_endofsentence_streaming_tokens(self): + """Test the specific use case of streaming LLM tokens.""" + + # These are the scenarios that were problematic with the original regex + # Single tokens should not be considered complete sentences + assert not match_endofsentence("Hello"), "Single token should not be sentence" + assert not match_endofsentence("world"), "Single token should not be sentence" + assert not match_endofsentence("The"), "Single token should not be sentence" + assert not match_endofsentence("quick"), "Single token should not be sentence" + + # But accumulating tokens should eventually form sentences + assert not match_endofsentence("Hello world"), "No punctuation = incomplete" + assert match_endofsentence("Hello world.") == 12, "With punctuation = complete" + + # Test progressive building (simulating token streaming) + tokens = ["The", " quick", " brown", " fox", " jumps", "."] + accumulated = "" + for i, token in enumerate(tokens): + accumulated += token + if i < len(tokens) - 1: # All but the last token + assert not match_endofsentence(accumulated), ( + f"Should be incomplete at token {i}: '{accumulated}'" + ) + else: # Last token adds the period + assert match_endofsentence(accumulated) == len(accumulated), ( + f"Should be complete: '{accumulated}'" + ) + + # Test with multiple sentences + assert match_endofsentence("First sentence. Second incomplete") == 15, ( + "Should return end of first sentence" + ) + class TestStartEndTags(unittest.IsolatedAsyncioTestCase): async def test_empty(self):