de58f30014
- Add new models for `KnowledgeDocument` and `KnowledgeChunk` to manage document ingestion and chunking. - Implement S3-compatible storage integration for knowledge documents, allowing for file uploads and retrieval. - Introduce API endpoints for managing knowledge bases and documents, including creation, deletion, and searching. - Update frontend components to support knowledge base configuration and document management, improving user interaction. - Enhance backend services for knowledge processing and retrieval, ensuring robust handling of document statuses and errors.
24 lines
796 B
Python
24 lines
796 B
Python
import unittest
|
|
|
|
from services.knowledge import extract_text, split_text
|
|
|
|
|
|
class KnowledgeTextTest(unittest.TestCase):
|
|
def test_extracts_utf8_text(self):
|
|
self.assertEqual(extract_text("说明.txt", "你好，知识库".encode()), "你好，知识库")
|
|
|
|
def test_split_text_keeps_overlap_and_all_content(self):
|
|
text = "第一段。" * 250
|
|
chunks = split_text(text, chunk_size=120, overlap=20)
|
|
self.assertGreater(len(chunks), 1)
|
|
self.assertTrue(all(chunk for chunk in chunks))
|
|
self.assertLessEqual(max(map(len, chunks)), 120)
|
|
|
|
def test_rejects_unsupported_binary_file(self):
|
|
with self.assertRaisesRegex(ValueError, "暂仅支持"):
|
|
extract_text("archive.zip", b"data")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|