Update match_endofsentence to use NLTK sentence tokenizer

CHANGELOG.md

@@ -20,6 +20,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Updated `MiniMaxHttpTTSService` with a `base_url` arg where you can specify
   the Global endpoint (default) or Mainland China.
 
+- Replaced regex-based sentence detection in `match_endofsentence` with NLTK's
+  punkt_tab tokenizer for more reliable sentence boundary detection.
+
 - Changed the `livekit` optional dependency for `tenacity` to
   `tenacity>=8.2.3,<10.0.0` in order to support the `google-genai` package.
 

pyproject.toml

@@ -25,6 +25,7 @@ dependencies = [
     "docstring_parser~=0.16",
     "loguru~=0.7.3",
     "Markdown~=3.7",
+    "nltk>=3.9.1",
     "numpy>=1.26.4",
     "Pillow~=11.1.0",
     "protobuf~=5.29.3",

src/pipecat/utils/string.py

@@ -9,29 +9,72 @@
 This module provides utilities for natural language text processing including
 sentence boundary detection, email and number pattern handling, and XML-style
 tag parsing for structured text content.
+
+Dependencies:
+    This module uses NLTK (Natural Language Toolkit) for robust sentence
+    tokenization. NLTK is licensed under the Apache License 2.0.
+    See: https://www.nltk.org/
+    Source: https://www.nltk.org/api/nltk.tokenize.punkt.html
 """
 
 import re
-from typing import Optional, Sequence, Tuple
+from typing import FrozenSet, Optional, Sequence, Tuple
 
-ENDOFSENTENCE_PATTERN_STR = r"""
-    (?<![A-Z])       # Negative lookbehind: not preceded by an uppercase letter (e.g., "U.S.A.")
-    (?<!\d\.\d)      # Not preceded by a decimal number (e.g., "3.14159")
-    (?<!^\d\.)       # Not preceded by a numbered list item (e.g., "1. Let's start")
-    (?<!\d\s[ap])    # Negative lookbehind: not preceded by time (e.g., "3:00 a.m.")
-    (?<!Mr|Ms|Dr)    # Negative lookbehind: not preceded by Mr, Ms, Dr (combined bc. length is the same)
-    (?<!Mrs)         # Negative lookbehind: not preceded by "Mrs"
-    (?<!Prof)        # Negative lookbehind: not preceded by "Prof"
-    (\.\s*\.\s*\.|[\.\?\!;])|   # Match a period, question mark, exclamation point, or semicolon
-    (\。\s*\。\s*\。|[。？！；।])  # the full-width version (mainly used in East Asian languages such as Chinese, Hindi)
-    $                # End of string
-"""
+import nltk
+from nltk.tokenize import sent_tokenize
 
-ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)
+# Ensure punkt_tab tokenizer data is available
+try:
+    nltk.data.find("tokenizers/punkt_tab")
+except LookupError:
+    nltk.download("punkt_tab", quiet=True)
 
-EMAIL_PATTERN = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
-
-NUMBER_PATTERN = re.compile(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?")
+SENTENCE_ENDING_PUNCTUATION: FrozenSet[str] = frozenset(
+    {
+        # Latin script punctuation (most European languages, Filipino, etc.)
+        ".",
+        "!",
+        "?",
+        ";",
+        # East Asian punctuation (Chinese (Traditional & Simplified), Japanese, Korean)
+        "。",  # Ideographic full stop
+        "？",  # Full-width question mark
+        "！",  # Full-width exclamation mark
+        "；",  # Full-width semicolon
+        "．",  # Full-width period
+        "｡",  # Halfwidth ideographic period
+        # Indic scripts punctuation (Hindi, Sanskrit, Marathi, Nepali, Bengali, Tamil, Telugu, Kannada, Malayalam, Gujarati, Punjabi, Oriya, Assamese)
+        "।",  # Devanagari danda (single vertical bar)
+        "॥",  # Devanagari double danda (double vertical bar)
+        # Arabic script punctuation (Arabic, Persian, Urdu, Pashto)
+        "؟",  # Arabic question mark
+        "؛",  # Arabic semicolon
+        "۔",  # Urdu full stop
+        "؏",  # Arabic sign misra (classical texts)
+        # Thai
+        "।",  # Thai uses Devanagari-style punctuation in some contexts
+        # Myanmar/Burmese
+        "၊",  # Myanmar sign little section
+        "။",  # Myanmar sign section
+        # Khmer
+        "។",  # Khmer sign khan
+        "៕",  # Khmer sign bariyoosan
+        # Lao
+        "໌",  # Lao cancellation mark (used as period)
+        "༎",  # Tibetan mark delimiter tsheg bstar (also used in Lao contexts)
+        # Tibetan
+        "།",  # Tibetan mark intersyllabic tsheg
+        "༎",  # Tibetan mark delimiter tsheg bstar
+        # Armenian
+        "։",  # Armenian full stop
+        "՜",  # Armenian exclamation mark
+        "՞",  # Armenian question mark
+        # Ethiopic script (Amharic)
+        "።",  # Ethiopic full stop
+        "፧",  # Ethiopic question mark
+        "፨",  # Ethiopic paragraph separator
+    }
+)
 
 StartEndTags = Tuple[str, str]
 
@@ -58,10 +101,9 @@ def replace_match(text: str, match: re.Match, old: str, new: str) -> str:
 def match_endofsentence(text: str) -> int:
     """Find the position of the end of a sentence in the provided text.
 
-    This function processes the input text by replacing periods in email
-    addresses and numbers with ampersands to prevent them from being
-    misidentified as sentence terminals. It then searches for the end of a
-    sentence using a specified regex pattern.
+    This function uses NLTK's sentence tokenizer to detect sentence boundaries
+    in the input text, combined with punctuation verification to ensure that
+    single tokens without proper sentence endings aren't considered complete sentences.
 
     Args:
         text: The input text in which to find the end of the sentence.
@@ -71,21 +113,33 @@ def match_endofsentence(text: str) -> int:
     """
     text = text.rstrip()
 
-    # Replace email dots by ampersands so we can find the end of sentence. For
-    # example, first.last@email.com becomes first&last@email&com.
-    emails = list(EMAIL_PATTERN.finditer(text))
-    for email_match in emails:
-        text = replace_match(text, email_match, ".", "&")
+    if not text:
+        return 0
 
-    # Replace number dots by ampersands so we can find the end of sentence.
-    numbers = list(NUMBER_PATTERN.finditer(text))
-    for number_match in numbers:
-        text = replace_match(text, number_match, ".", "&")
+    # Use NLTK's sentence tokenizer to find sentence boundaries
+    sentences = sent_tokenize(text)
 
-    # Match against the new text.
-    match = ENDOFSENTENCE_PATTERN.search(text)
+    if not sentences:
+        return 0
 
-    return match.end() if match else 0
+    first_sentence = sentences[0]
+
+    # If there's only one sentence that equals the entire text,
+    # verify it actually ends with sentence-ending punctuation.
+    # This is required as NLTK may return a single sentence for
+    # text that's a single word. In the case of LLM tokens, it's
+    # common for text to be single words, so we need to ensure
+    # sentence-ending punctuation is present.
+    if len(sentences) == 1 and first_sentence == text:
+        return len(text) if text and text[-1] in SENTENCE_ENDING_PUNCTUATION else 0
+
+    # If there are multiple sentences, the first one is complete by definition
+    # (NLTK found a boundary, so there must be proper punctuation)
+    if len(sentences) > 1:
+        return len(first_sentence)
+
+    # Single sentence that doesn't equal the full text means incomplete
+    return 0
 
 
 def parse_start_end_tags(

tests/test_utils_string.py

@@ -16,10 +16,13 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase):
         assert match_endofsentence("This is a sentence?") == 19
         assert match_endofsentence("This is a sentence;") == 19
         assert match_endofsentence("This is a sentence...") == 21
-        assert match_endofsentence("This is a sentence . . .") == 24
-        assert match_endofsentence("This is a sentence. ..") == 22
+        assert match_endofsentence("This is a sentence. This is another one") == 19
         assert match_endofsentence("This is for Mr. and Mrs. Jones.") == 31
-        assert match_endofsentence("U.S.A and U.S.A..") == 17
+        assert match_endofsentence("Meet the new Mr. and Mrs.") == 25
+        assert match_endofsentence("U.S.A. and N.A.S.A.") == 19
+        assert match_endofsentence("USA and NASA.") == 13
+        assert match_endofsentence("My number is 123-456-7890.") == 26
+        assert match_endofsentence("For information, call 411.") == 26
         assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48
         assert match_endofsentence("My email is foo.bar@pipecat.ai.") == 31
         assert match_endofsentence("My email is spell(foo.bar@pipecat.ai).") == 38
@@ -27,41 +30,162 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase):
         assert match_endofsentence("The number pi is 3.14159.") == 25
         assert match_endofsentence("Valid scientific notation 1.23e4.") == 33
         assert match_endofsentence("Valid scientific notation 0.e4.") == 31
+        assert match_endofsentence("It still early, it's 3:00 a.m.") == 30
         assert not match_endofsentence("This is not a sentence")
         assert not match_endofsentence("This is not a sentence,")
         assert not match_endofsentence("This is not a sentence, ")
         assert not match_endofsentence("Ok, Mr. Smith let's ")
         assert not match_endofsentence("Dr. Walker, I presume ")
         assert not match_endofsentence("Prof. Walker, I presume ")
-        assert not match_endofsentence("zweitens, und 3.")
-        assert not match_endofsentence("Heute ist Dienstag, der 3.")  # 3. Juli 2024
-        assert not match_endofsentence("America, or the U.")  # U.S.A.
-        assert not match_endofsentence("It still early, it's 3:00 a.")  # 3:00 a.m.
+        assert not match_endofsentence("zweitens, und 3")
+        assert not match_endofsentence("Heute ist Dienstag, der 3")  # 3. Juli 2024
+        assert not match_endofsentence("America, or the U.S")  # U.S.A.
         assert not match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai")
         assert not match_endofsentence("The number pi is 3.14159")
 
-    async def test_endofsentence_zh(self):
+    async def test_endofsentence_multilingual(self):
+        """Test sentence detection across various language families and scripts."""
+
+        # Arabic script (Arabic, Urdu, Persian)
+        arabic_sentences = [
+            "مرحبا؟",  # Arabic question mark
+            "السلام عليكم؛",  # Arabic semicolon
+            "یہ اردو ہے۔",  # Urdu full stop
+        ]
+        for sentence in arabic_sentences:
+            assert match_endofsentence(sentence), f"Failed for Arabic/Urdu: {sentence}"
+
+        # Should not match incomplete Arabic
+        assert not match_endofsentence("مرحبا،"), "Arabic comma should not end sentence"
+
         chinese_sentences = [
             "你好。",
             "你好！",
             "吃了吗？",
             "安全第一；",
         ]
-        for i in chinese_sentences:
-            assert match_endofsentence(i)
+        for sentence in chinese_sentences:
+            assert match_endofsentence(sentence), f"Failed for Chinese: {sentence}"
         assert not match_endofsentence("你好，")
 
-    async def test_endofsentence_hi(self):
         hindi_sentences = [
             "हैलो।",
             "हैलो！",
             "आप खाये हैं？",
             "सुरक्षा पहले।",
         ]
-        for i in hindi_sentences:
-            assert match_endofsentence(i)
+        for sentence in hindi_sentences:
+            assert match_endofsentence(sentence), f"Failed for Hindi: {sentence}"
         assert not match_endofsentence("हैलो，")
 
+        # East Asian (Japanese, Korean)
+        japanese_sentences = [
+            "こんにちは。",  # Japanese
+            "元気ですか？",  # Japanese question
+            "ありがとう！",  # Japanese exclamation
+        ]
+        for sentence in japanese_sentences:
+            assert match_endofsentence(sentence), f"Failed for Japanese: {sentence}"
+
+        korean_sentences = [
+            "안녕하세요。",  # Korean with ideographic period
+            "어떻게 지내세요？",  # Korean question
+        ]
+        for sentence in korean_sentences:
+            assert match_endofsentence(sentence), f"Failed for Korean: {sentence}"
+
+        # Southeast Asian scripts
+        thai_sentences = [
+            "สวัสดี।",  # Thai with Devanagari-style punctuation
+        ]
+        for sentence in thai_sentences:
+            assert match_endofsentence(sentence), f"Failed for Thai: {sentence}"
+
+        myanmar_sentences = [
+            "မင်္ဂလာပါ၊",  # Myanmar little section
+            "ကျေးဇူးတင်ပါတယ်။",  # Myanmar section
+        ]
+        for sentence in myanmar_sentences:
+            assert match_endofsentence(sentence), f"Failed for Myanmar: {sentence}"
+
+        # Other Indic scripts (same punctuation as Hindi but different scripts)
+        bengali_sentences = [
+            "নমস্কার।",  # Bengali
+            "আপনি কেমন আছেন？",  # Bengali question (uses Latin ?)
+        ]
+        for sentence in bengali_sentences:
+            assert match_endofsentence(sentence), f"Failed for Bengali: {sentence}"
+
+        tamil_sentences = [
+            "வணக்கம்।",  # Tamil
+            "நீங்கள் எப்படி இருக்கிறீர்கள்？",  # Tamil question
+        ]
+        for sentence in tamil_sentences:
+            assert match_endofsentence(sentence), f"Failed for Tamil: {sentence}"
+
+        # Armenian
+        armenian_sentences = [
+            "Բարև։",  # Armenian full stop
+            "Ինչպես եք՞",  # Armenian question mark
+            "Շնորհակալություն՜",  # Armenian exclamation
+        ]
+        for sentence in armenian_sentences:
+            assert match_endofsentence(sentence), f"Failed for Armenian: {sentence}"
+
+        # Ethiopic (Amharic)
+        amharic_sentences = [
+            "ሰላም።",  # Ethiopic full stop
+            "እንዴት ነዎት፧",  # Ethiopic question mark
+        ]
+        for sentence in amharic_sentences:
+            assert match_endofsentence(sentence), f"Failed for Amharic: {sentence}"
+
+        # Languages using Latin punctuation (should still work)
+        latin_script_sentences = [
+            "Hola.",  # Spanish
+            "Bonjour!",  # French
+            "Guten Tag?",  # German
+            "Привет.",  # Russian (Cyrillic but uses Latin punctuation)
+            "Γεια σας.",  # Greek
+            "שלום.",  # Hebrew
+            "გამარჯობა.",  # Georgian
+        ]
+        for sentence in latin_script_sentences:
+            assert match_endofsentence(sentence), f"Failed for Latin script: {sentence}"
+
+    async def test_endofsentence_streaming_tokens(self):
+        """Test the specific use case of streaming LLM tokens."""
+
+        # These are the scenarios that were problematic with the original regex
+        # Single tokens should not be considered complete sentences
+        assert not match_endofsentence("Hello"), "Single token should not be sentence"
+        assert not match_endofsentence("world"), "Single token should not be sentence"
+        assert not match_endofsentence("The"), "Single token should not be sentence"
+        assert not match_endofsentence("quick"), "Single token should not be sentence"
+
+        # But accumulating tokens should eventually form sentences
+        assert not match_endofsentence("Hello world"), "No punctuation = incomplete"
+        assert match_endofsentence("Hello world.") == 12, "With punctuation = complete"
+
+        # Test progressive building (simulating token streaming)
+        tokens = ["The", " quick", " brown", " fox", " jumps", "."]
+        accumulated = ""
+        for i, token in enumerate(tokens):
+            accumulated += token
+            if i < len(tokens) - 1:  # All but the last token
+                assert not match_endofsentence(accumulated), (
+                    f"Should be incomplete at token {i}: '{accumulated}'"
+                )
+            else:  # Last token adds the period
+                assert match_endofsentence(accumulated) == len(accumulated), (
+                    f"Should be complete: '{accumulated}'"
+                )
+
+        # Test with multiple sentences
+        assert match_endofsentence("First sentence. Second incomplete") == 15, (
+            "Should return end of first sentence"
+        )
+
 
 class TestStartEndTags(unittest.IsolatedAsyncioTestCase):
     async def test_empty(self):