diff --git a/CHANGELOG.md b/CHANGELOG.md index 249474080..6e626a7b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -138,6 +138,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Fixed a `match_endofsentence` issue that would result in emails to be + considered an end of sentence. + - Fixed an issue where the RTVI message `disconnect-bot` was pushing an `EndFrame`, resulting in the pipeline not shutting down. It now pushes an `EndTaskFrame` upstream to shutdown the pipeline. diff --git a/src/pipecat/utils/string.py b/src/pipecat/utils/string.py index 154d03174..06f2fc175 100644 --- a/src/pipecat/utils/string.py +++ b/src/pipecat/utils/string.py @@ -17,9 +17,26 @@ ENDOFSENTENCE_PATTERN_STR = r""" (\。\s*\。\s*\。|[。?!;।]) # the full-width version (mainly used in East Asian languages such as Chinese, Hindi) $ # End of string """ + ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE) +EMAIL_PATTERN = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}") + def match_endofsentence(text: str) -> int: - match = ENDOFSENTENCE_PATTERN.search(text.rstrip()) + text = text.rstrip() + + # Find all emails. + emails = list(EMAIL_PATTERN.finditer(text)) + + # Replace email dots by ampersands so we can find the end of sentence. + for email_match in emails: + start = email_match.start() + end = email_match.end() + new_email = text[start:end].replace(".", "&") + text = text[:start] + new_email + text[end:] + + # Match against the new text. + match = ENDOFSENTENCE_PATTERN.search(text) + return match.end() if match else 0 diff --git a/tests/test_utils_string.py b/tests/test_utils_string.py index e4cdb4cb4..24519f724 100644 --- a/tests/test_utils_string.py +++ b/tests/test_utils_string.py @@ -18,6 +18,9 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert match_endofsentence("This is a sentence...") == 21 assert match_endofsentence("This is a sentence . . .") == 24 assert match_endofsentence("This is a sentence. ..") == 22 + assert match_endofsentence("This is for Mr. and Mrs. Jones.") == 31 + assert match_endofsentence("U.S.A and U.S.A..") == 17 + assert match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai.") == 48 assert not match_endofsentence("This is not a sentence") assert not match_endofsentence("This is not a sentence,") assert not match_endofsentence("This is not a sentence, ") @@ -28,6 +31,7 @@ class TestUtilsString(unittest.IsolatedAsyncioTestCase): assert not match_endofsentence("Heute ist Dienstag, der 3.") # 3. Juli 2024 assert not match_endofsentence("America, or the U.") # U.S.A. assert not match_endofsentence("It still early, it's 3:00 a.") # 3:00 a.m. + assert not match_endofsentence("My emails are foo@pipecat.ai and bar@pipecat.ai") async def test_endofsentence_zh(self): chinese_sentences = [