Merge pull request #494 from badbye/full-width-punctuations

add full-width punctuations as end of the sentence

tests/test_ai_services.py

@@ -29,6 +29,7 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
         assert match_endofsentence("This is a sentence! ")
         assert match_endofsentence("This is a sentence?")
         assert match_endofsentence("This is a sentence:")
+        assert match_endofsentence("This is a sentence;")
         assert not match_endofsentence("This is not a sentence")
         assert not match_endofsentence("This is not a sentence,")
         assert not match_endofsentence("This is not a sentence, ")
@@ -40,6 +41,18 @@ class TestBaseAIService(unittest.IsolatedAsyncioTestCase):
         assert not match_endofsentence("America, or the U.")  # U.S.A.
         assert not match_endofsentence("It still early, it's 3:00 a.")  # 3:00 a.m.
 
+    async def test_endofsentence_zh(self):
+        chinese_sentences = [
+            "你好。",
+            "你好！",
+            "吃了吗？",
+            "安全第一；",
+            "他说：",
+        ]
+        for i in chinese_sentences:
+            assert match_endofsentence(i)
+        assert not match_endofsentence("你好，")
+
 
 if __name__ == "__main__":
     unittest.main()