utils(string): support email addresses in end of sentence matching
This commit is contained in:
@@ -17,9 +17,26 @@ ENDOFSENTENCE_PATTERN_STR = r"""
|
||||
(\。\s*\。\s*\。|[。?!;।]) # the full-width version (mainly used in East Asian languages such as Chinese, Hindi)
|
||||
$ # End of string
|
||||
"""
|
||||
|
||||
ENDOFSENTENCE_PATTERN = re.compile(ENDOFSENTENCE_PATTERN_STR, re.VERBOSE)
|
||||
|
||||
EMAIL_PATTERN = re.compile(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}")
|
||||
|
||||
|
||||
def match_endofsentence(text: str) -> int:
|
||||
match = ENDOFSENTENCE_PATTERN.search(text.rstrip())
|
||||
text = text.rstrip()
|
||||
|
||||
# Find all emails.
|
||||
emails = list(EMAIL_PATTERN.finditer(text))
|
||||
|
||||
# Replace email dots by ampersands so we can find the end of sentence.
|
||||
for email_match in emails:
|
||||
start = email_match.start()
|
||||
end = email_match.end()
|
||||
new_email = text[start:end].replace(".", "&")
|
||||
text = text[:start] + new_email + text[end:]
|
||||
|
||||
# Match against the new text.
|
||||
match = ENDOFSENTENCE_PATTERN.search(text)
|
||||
|
||||
return match.end() if match else 0
|
||||
|
||||
Reference in New Issue
Block a user