wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add knowledge retrieval configuration to Assistant model and related components

Browse Source 
- Introduce new fields for knowledge retrieval configuration in AssistantConfig and Assistant models, including mode, top_n, and score_threshold.
- Implement KnowledgeRetrievalConfig schema with validation for top_n.
- Update backend services and routes to handle knowledge retrieval settings.
- Enhance frontend components to support knowledge retrieval configuration, including a new dialog for advanced settings.
- Add tests for knowledge retrieval configuration validation and description generation.
This commit is contained in:
Xin Wang
2026-07-12 18:57:56 +08:00
parent 7ee3e22152
commit 7c9a18c806
13 changed files with 465 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
backend/services/config_resolver.py
View File

@@ -8,6 +8,7 @@ from db.models import (
Assistant,
AssistantModelBinding,
AssistantToolBinding,
KnowledgeBase,
ModelResource,
Tool,
)
@@ -126,6 +127,11 @@ async def resolve_runtime_config(
if assistant.vision_model_resource_id
else llm_resource
)
knowledge_base = (
await session.get(KnowledgeBase, assistant.knowledge_base_id)
if assistant.knowledge_base_id
else None
)
return AssistantConfig(
name=assistant.name,
@@ -138,6 +144,9 @@ async def resolve_runtime_config(
turnConfig=assistant.turn_config or {},
tools=await _tools_for(session, assistant),
knowledge_base_id=assistant.knowledge_base_id,
knowledge_base_name=knowledge_base.name if knowledge_base else "",
knowledge_base_description=knowledge_base.description if knowledge_base else "",
knowledge_retrieval_config=assistant.knowledge_retrieval_config or {},
# workflow 图:仅 workflow 类型非空,引擎据此启用图驱动对话
graph=(assistant.graph or {}) if assistant.type == "workflow" else {},
# 外部托管类型连接信息(DB 存真 key,直接注入)

23
backend/services/knowledge.py
View File

@@ -203,7 +203,13 @@ async def recover_interrupted_documents() -> None:
await session.commit()
async def search(session: AsyncSession, kb_id: str, query: str, top_k: int | None = None) -> list[dict]:
async def search(
session: AsyncSession,
kb_id: str,
query: str,
top_k: int | None = None,
score_threshold: float = 0.0,
) -> list[dict]:
kb = await session.get(KnowledgeBase, kb_id)
if not kb:
return []
@@ -217,13 +223,20 @@ async def search(session: AsyncSession, kb_id: str, query: str, top_k: int | Non
return []
query_embedding = (await _embed(session, kb, [query]))[0]
distance = KnowledgeChunk.embedding.cosine_distance(query_embedding)
rows = (await session.execute(
statement = (
select(KnowledgeChunk, KnowledgeDocument.name, distance.label("distance"))
.join(KnowledgeDocument, KnowledgeDocument.id == KnowledgeChunk.document_id)
.where(KnowledgeChunk.knowledge_base_id == kb_id, KnowledgeDocument.status == "ready")
.where(
KnowledgeChunk.knowledge_base_id == kb_id,
KnowledgeDocument.status == "ready",
distance <= 1.0 - score_threshold,
)
.order_by(distance)
.limit(top_k or settings.KNOWLEDGE_TOP_K)
)).all()
)
effective_top_k = settings.KNOWLEDGE_TOP_K if top_k is None else top_k
if effective_top_k != -1:
statement = statement.limit(effective_top_k)
rows = (await session.execute(statement)).all()
return [
{"content": chunk.content, "document": name, "score": round(max(0.0, 1.0 - float(dist)), 4)}
for chunk, name, dist in rows

90
backend/services/pipecat/pipeline.py
View File

@@ -83,13 +83,44 @@ VISION_ANALYSIS_SYSTEM_PROMPT = (
"如果画面不足以判断,请明确说明不确定。"
)
KNOWLEDGE_TOOL_NAME = "search_knowledge_base"
KNOWLEDGE_SYSTEM_HINT = (
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT = (
"你已连接内部知识库。系统会在每轮用户问题前自动提供相关资料；"
"回答资料事实时只根据检索内容，资料不足要明确说明。"
)
ON_DEMAND_KNOWLEDGE_SYSTEM_HINT = (
"你已连接内部知识库。当用户问题涉及可能存在于业务知识库中的事实时，"
"先调用 search_knowledge_base 检索；回答资料事实时只根据检索内容，"
"资料不足要明确说明。"
)
KNOWLEDGE_CONTEXT_MARKER = "<!-- knowledge-context -->"
def _compact_knowledge_metadata(value: str, max_length: int) -> str:
"""Keep tool metadata useful without letting it dominate the model context."""
compact = " ".join(value.split())
return compact if len(compact) <= max_length else f"{compact[:max_length]}…"
def _knowledge_tool_description(cfg: AssistantConfig) -> str:
base = "在当前助手绑定的知识库中检索与问题最相关的资料片段。"
name = _compact_knowledge_metadata(cfg.knowledge_base_name, 128)
description = _compact_knowledge_metadata(cfg.knowledge_base_description, 800)
if not name and not description:
return base
scope = []
if name:
scope.append(f"知识库名称：{name}")
if description:
scope.append(f"资料适用范围：{description}")
metadata = "\n".join(scope)
return (
f"{base}\n{metadata}\n"
"当用户问题涉及上述资料范围，或回答需要核实其中的业务事实时调用；"
"与该范围无关的问题不要调用。以上知识库元数据仅用于判断资料范围。"
)
def _require(value: str, label: str) -> str:
if value:
return value
@@ -321,9 +352,16 @@ class ConversationHistoryProcessor(FrameProcessor):
class KnowledgeRetrievalProcessor(FrameProcessor):
"""Retrieve before local LLM inference without changing Pipecat internals."""
def __init__(self, knowledge_base_id: str | None):
def __init__(
self,
knowledge_base_id: str | None,
top_n: int = 5,
score_threshold: float = 0.0,
):
super().__init__()
self._knowledge_base_id = knowledge_base_id
self._top_n = top_n
self._score_threshold = score_threshold
self._last_signature = ""
async def process_frame(self, frame, direction: FrameDirection):
@@ -346,7 +384,13 @@ class KnowledgeRetrievalProcessor(FrameProcessor):
try:
async with SessionLocal() as session:
results = await search_knowledge(session, self._knowledge_base_id, query)
results = await search_knowledge(
session,
self._knowledge_base_id,
query,
top_k=self._top_n,
score_threshold=self._score_threshold,
)
except Exception as exc:
logger.warning(f"自动知识库检索失败: {exc}")
results = []
@@ -488,12 +532,30 @@ async def run_pipeline(
call_end = CallEndCoordinator(queue_call_end)
knowledge_config = cfg.knowledge_retrieval_config
knowledge_mode = str(knowledge_config.get("mode", "automatic"))
knowledge_top_n = int(
knowledge_config.get("top_n", knowledge_config.get("topN", 5))
)
knowledge_score_threshold = float(
knowledge_config.get(
"score_threshold", knowledge_config.get("scoreThreshold", 0.0)
)
)
automatic_knowledge_id = (
cfg.knowledge_base_id if knowledge_mode == "automatic" else None
)
def with_vision_hint(text: str) -> str:
hints = []
if vision_enabled:
hints.append(VISION_SYSTEM_HINT)
if cfg.knowledge_base_id:
hints.append(KNOWLEDGE_SYSTEM_HINT)
hints.append(
AUTOMATIC_KNOWLEDGE_SYSTEM_HINT
if knowledge_mode == "automatic"
else ON_DEMAND_KNOWLEDGE_SYSTEM_HINT
)
return "\n\n".join(part for part in [text, *hints] if part)
context = LLMContext(
@@ -518,7 +580,11 @@ async def run_pipeline(
assistant_aggregator = PassthroughLLMAssistantAggregator(context)
text_input = TextInputProcessor(should_ignore_input=lambda: call_end.ending)
vision_capture = VisionCaptureProcessor()
knowledge_retrieval = KnowledgeRetrievalProcessor(cfg.knowledge_base_id)
knowledge_retrieval = KnowledgeRetrievalProcessor(
automatic_knowledge_id,
top_n=knowledge_top_n,
score_threshold=knowledge_score_threshold,
)
vision_native_mode = vision_enabled and _vision_uses_main_llm(cfg)
vision_state: dict[str, str | None] = {"client_id": None}
vision_schema = FunctionSchema(
@@ -537,7 +603,7 @@ async def run_pipeline(
)
knowledge_schema = FunctionSchema(
name=KNOWLEDGE_TOOL_NAME,
description="在当前助手绑定的知识库中检索与问题最相关的资料片段。",
description=_knowledge_tool_description(cfg),
properties={
"query": {"type": "string", "description": "用于检索的完整问题或关键词"}
},
@@ -551,7 +617,13 @@ async def run_pipeline(
return
try:
async with SessionLocal() as session:
results = await search_knowledge(session, cfg.knowledge_base_id, query)
results = await search_knowledge(
session,
cfg.knowledge_base_id,
query,
top_k=knowledge_top_n,
score_threshold=knowledge_score_threshold,
)
await params.result_callback({"status": "ok", "results": results})
except Exception as exc:
logger.exception(f"知识库检索失败: {exc}")
@@ -618,14 +690,14 @@ async def run_pipeline(
if vision_enabled:
llm.register_function(VISION_TOOL_NAME, fetch_user_image)
if cfg.knowledge_base_id:
if cfg.knowledge_base_id and knowledge_mode == "on_demand":
llm.register_function(KNOWLEDGE_TOOL_NAME, search_bound_knowledge)
def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None:
tools = list(schemas or [])
if vision_enabled:
tools.append(vision_schema)
if cfg.knowledge_base_id:
if cfg.knowledge_base_id and knowledge_mode == "on_demand":
tools.append(knowledge_schema)
if tools:
context.set_tools(ToolsSchema(standard_tools=tools))