Enhance knowledge base functionality and integrate S3 storage support

- Add new models for `KnowledgeDocument` and `KnowledgeChunk` to manage document ingestion and chunking.
- Implement S3-compatible storage integration for knowledge documents, allowing for file uploads and retrieval.
- Introduce API endpoints for managing knowledge bases and documents, including creation, deletion, and searching.
- Update frontend components to support knowledge base configuration and document management, improving user interaction.
- Enhance backend services for knowledge processing and retrieval, ensuring robust handling of document statuses and errors.

backend/migrations/versions/20260712_0005_add_knowledge_documents.py

@@ -0,0 +1,55 @@
+"""add knowledge documents and vector chunks
+
+Revision ID: 20260712_0005
+Revises: 20260712_0004
+"""
+from collections.abc import Sequence
+
+from alembic import op
+import sqlalchemy as sa
+from pgvector.sqlalchemy import Vector
+
+revision: str = "20260712_0005"
+down_revision: str | Sequence[str] | None = "20260712_0004"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    op.execute("CREATE EXTENSION IF NOT EXISTS vector")
+    op.create_table(
+        "knowledge_documents",
+        sa.Column("id", sa.String(40), primary_key=True),
+        sa.Column("knowledge_base_id", sa.String(40), nullable=False),
+        sa.Column("name", sa.String(255), nullable=False),
+        sa.Column("source_type", sa.String(16), nullable=False),
+        sa.Column("mime_type", sa.String(128), server_default="text/plain", nullable=False),
+        sa.Column("storage_key", sa.String(1024), nullable=True),
+        sa.Column("size_bytes", sa.Integer(), server_default="0", nullable=False),
+        sa.Column("status", sa.String(16), server_default="processing", nullable=False),
+        sa.Column("error_message", sa.String(2048), server_default="", nullable=False),
+        sa.Column("chunk_count", sa.Integer(), server_default="0", nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.ForeignKeyConstraint(["knowledge_base_id"], ["knowledge_bases.id"], ondelete="CASCADE"),
+    )
+    op.create_index("ix_knowledge_documents_knowledge_base_id", "knowledge_documents", ["knowledge_base_id"])
+    op.create_table(
+        "knowledge_chunks",
+        sa.Column("id", sa.String(40), primary_key=True),
+        sa.Column("knowledge_base_id", sa.String(40), nullable=False),
+        sa.Column("document_id", sa.String(40), nullable=False),
+        sa.Column("chunk_index", sa.Integer(), nullable=False),
+        sa.Column("content", sa.Text(), nullable=False),
+        sa.Column("embedding", Vector(), nullable=False),
+        sa.ForeignKeyConstraint(["knowledge_base_id"], ["knowledge_bases.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["document_id"], ["knowledge_documents.id"], ondelete="CASCADE"),
+        sa.UniqueConstraint("document_id", "chunk_index", name="uq_knowledge_chunk_position"),
+    )
+    op.create_index("ix_knowledge_chunks_knowledge_base_id", "knowledge_chunks", ["knowledge_base_id"])
+    op.create_index("ix_knowledge_chunks_document_id", "knowledge_chunks", ["document_id"])
+
+
+def downgrade() -> None:
+    op.drop_table("knowledge_chunks")
+    op.drop_table("knowledge_documents")