de58f30014
- Add new models for `KnowledgeDocument` and `KnowledgeChunk` to manage document ingestion and chunking. - Implement S3-compatible storage integration for knowledge documents, allowing for file uploads and retrieval. - Introduce API endpoints for managing knowledge bases and documents, including creation, deletion, and searching. - Update frontend components to support knowledge base configuration and document management, improving user interaction. - Enhance backend services for knowledge processing and retrieval, ensuring robust handling of document statuses and errors.
56 lines
2.6 KiB
Python
56 lines
2.6 KiB
Python
"""add knowledge documents and vector chunks
|
|
|
|
Revision ID: 20260712_0005
|
|
Revises: 20260712_0004
|
|
"""
|
|
from collections.abc import Sequence
|
|
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
from pgvector.sqlalchemy import Vector
|
|
|
|
revision: str = "20260712_0005"
|
|
down_revision: str | Sequence[str] | None = "20260712_0004"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.execute("CREATE EXTENSION IF NOT EXISTS vector")
|
|
op.create_table(
|
|
"knowledge_documents",
|
|
sa.Column("id", sa.String(40), primary_key=True),
|
|
sa.Column("knowledge_base_id", sa.String(40), nullable=False),
|
|
sa.Column("name", sa.String(255), nullable=False),
|
|
sa.Column("source_type", sa.String(16), nullable=False),
|
|
sa.Column("mime_type", sa.String(128), server_default="text/plain", nullable=False),
|
|
sa.Column("storage_key", sa.String(1024), nullable=True),
|
|
sa.Column("size_bytes", sa.Integer(), server_default="0", nullable=False),
|
|
sa.Column("status", sa.String(16), server_default="processing", nullable=False),
|
|
sa.Column("error_message", sa.String(2048), server_default="", nullable=False),
|
|
sa.Column("chunk_count", sa.Integer(), server_default="0", nullable=False),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
|
|
sa.ForeignKeyConstraint(["knowledge_base_id"], ["knowledge_bases.id"], ondelete="CASCADE"),
|
|
)
|
|
op.create_index("ix_knowledge_documents_knowledge_base_id", "knowledge_documents", ["knowledge_base_id"])
|
|
op.create_table(
|
|
"knowledge_chunks",
|
|
sa.Column("id", sa.String(40), primary_key=True),
|
|
sa.Column("knowledge_base_id", sa.String(40), nullable=False),
|
|
sa.Column("document_id", sa.String(40), nullable=False),
|
|
sa.Column("chunk_index", sa.Integer(), nullable=False),
|
|
sa.Column("content", sa.Text(), nullable=False),
|
|
sa.Column("embedding", Vector(), nullable=False),
|
|
sa.ForeignKeyConstraint(["knowledge_base_id"], ["knowledge_bases.id"], ondelete="CASCADE"),
|
|
sa.ForeignKeyConstraint(["document_id"], ["knowledge_documents.id"], ondelete="CASCADE"),
|
|
sa.UniqueConstraint("document_id", "chunk_index", name="uq_knowledge_chunk_position"),
|
|
)
|
|
op.create_index("ix_knowledge_chunks_knowledge_base_id", "knowledge_chunks", ["knowledge_base_id"])
|
|
op.create_index("ix_knowledge_chunks_document_id", "knowledge_chunks", ["document_id"])
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_table("knowledge_chunks")
|
|
op.drop_table("knowledge_documents")
|