utils(string): support email addresses in end of sentence matching

This commit is contained in:
Aleix Conchillo Flaqué
2025-02-26 17:23:06 -08:00
parent 2e0c6c2bd1
commit 1dbad2326a
3 changed files with 25 additions and 1 deletions

View File

@@ -17,9 +17,26 @@ ENDOFSENTENCE_PATTERN_STR = r"""
(\\s*\\s*\。|[。?!;।]) # the full-width version (mainly used in East Asian languages such as Chinese, Hindi)
$ # End of string
"""
ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)
EMAIL_PATTERN = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
def match_endofsentence(text: str) -> int:
match = ENDOFSENTENCE_PATTERN.search(text.rstrip())
text = text.rstrip()
# Find all emails.
emails = list(EMAIL_PATTERN.finditer(text))
# Replace email dots by ampersands so we can find the end of sentence.
for email_match in emails:
start = email_match.start()
end = email_match.end()
new_email = text[start:end].replace(".", "&")
text = text[:start] + new_email + text[end:]
# Match against the new text.
match = ENDOFSENTENCE_PATTERN.search(text)
return match.end() if match else 0