wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add reusable tools and assistant bindings

Browse Source 
- Introduce a new `Tool` model and `AssistantToolBinding` for managing reusable tools within the application.
- Implement CRUD operations for tools in the new `tools` route, allowing for the creation, retrieval, updating, and deletion of tools.
- Update the `Assistant` model to include a list of tool IDs, enabling assistants to utilize these tools.
- Enhance the backend routes to synchronize tool bindings with assistants, ensuring proper management of tool associations.
- Add frontend components for tool management, including a tool picker in the assistant configuration, improving user experience in tool selection.
- Create a mobile call page to facilitate video calls, integrating camera and microphone selection for enhanced communication capabilities.
- Update API definitions to include tool-related types and operations, ensuring consistency across the application.
- Add a migration script to create the necessary database tables for tools and bindings, supporting the new functionality.
This commit is contained in:
Xin Wang
2026-07-10 10:05:41 +08:00
parent 919325505a
commit 3ed9e1b388
14 changed files with 1815 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/app.py
View File

@@ -27,6 +27,7 @@ from routes import (
knowledge_bases,
model_registry,
node_types,
tools,
voice_webrtc,
voice_ws,
)
@@ -54,6 +55,7 @@ app.include_router(assistants.router)
app.include_router(knowledge_bases.router)
app.include_router(model_registry.router)
app.include_router(node_types.router)
app.include_router(tools.router)
app.include_router(voice_webrtc.router)
app.include_router(voice_ws.router)

42
backend/db/models.py
View File

@@ -145,3 +145,45 @@ class AssistantModelBinding(Base):
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
class Tool(Base):
"""Reusable LLM tool definition. Runtime execution is added separately."""
__tablename__ = "tools"
id: Mapped[str] = mapped_column(String(40), primary_key=True)
name: Mapped[str] = mapped_column(String(128))
function_name: Mapped[str] = mapped_column(String(64), unique=True, index=True)
type: Mapped[str] = mapped_column(String(24), index=True)
description: Mapped[str] = mapped_column(String(2048), default="")
definition: Mapped[dict] = mapped_column(JSONB, default=dict)
secrets: Mapped[dict] = mapped_column(JSONB, default=dict)
status: Mapped[str] = mapped_column(String(16), index=True, default="active")
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
class AssistantToolBinding(Base):
"""Prompt assistant to reusable tool many-to-many binding."""
__tablename__ = "assistant_tool_bindings"
assistant_id: Mapped[str] = mapped_column(
String(40),
ForeignKey("assistants.id", ondelete="CASCADE"),
primary_key=True,
)
tool_id: Mapped[str] = mapped_column(
String(40),
ForeignKey("tools.id", ondelete="RESTRICT"),
primary_key=True,
index=True,
)
created_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now()
)

72
backend/migrations/versions/20260710_0002_add_tools.py Normal file
View File

@@ -0,0 +1,72 @@
"""add reusable tools and assistant bindings
Revision ID: 20260710_0002
Revises: 20260709_0001
Create Date: 2026-07-10 00:00:00.000000
"""
from collections.abc import Sequence
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
revision: str = "20260710_0002"
down_revision: str | Sequence[str] | None = "20260709_0001"
branch_labels: str | Sequence[str] | None = None
depends_on: str | Sequence[str] | None = None
def upgrade() -> None:
op.create_table(
"tools",
sa.Column("id", sa.String(length=40), nullable=False),
sa.Column("name", sa.String(length=128), nullable=False),
sa.Column("function_name", sa.String(length=64), nullable=False),
sa.Column("type", sa.String(length=24), nullable=False),
sa.Column("description", sa.String(length=2048), server_default="", nullable=False),
sa.Column(
"definition",
postgresql.JSONB(astext_type=sa.Text()),
server_default=sa.text("'{}'::jsonb"),
nullable=False,
),
sa.Column(
"secrets",
postgresql.JSONB(astext_type=sa.Text()),
server_default=sa.text("'{}'::jsonb"),
nullable=False,
),
sa.Column("status", sa.String(length=16), server_default="active", nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_tools_function_name", "tools", ["function_name"], unique=True)
op.create_index("ix_tools_type", "tools", ["type"])
op.create_index("ix_tools_status", "tools", ["status"])
op.create_table(
"assistant_tool_bindings",
sa.Column("assistant_id", sa.String(length=40), nullable=False),
sa.Column("tool_id", sa.String(length=40), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
sa.ForeignKeyConstraint(["assistant_id"], ["assistants.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["tool_id"], ["tools.id"], ondelete="RESTRICT"),
sa.PrimaryKeyConstraint("assistant_id", "tool_id"),
)
op.create_index(
"ix_assistant_tool_bindings_tool_id",
"assistant_tool_bindings",
["tool_id"],
)
def downgrade() -> None:
op.drop_index("ix_assistant_tool_bindings_tool_id", table_name="assistant_tool_bindings")
op.drop_table("assistant_tool_bindings")
op.drop_index("ix_tools_status", table_name="tools")
op.drop_index("ix_tools_type", table_name="tools")
op.drop_index("ix_tools_function_name", table_name="tools")
op.drop_table("tools")

56
backend/routes/assistants.py
View File

@@ -2,7 +2,13 @@
import uuid
from db.models import Assistant, AssistantModelBinding, ModelResource
from db.models import (
Assistant,
AssistantModelBinding,
AssistantToolBinding,
ModelResource,
Tool,
)
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException
from schemas import AssistantOut, AssistantUpsert
@@ -83,6 +89,46 @@ async def _resource_ids(session: AsyncSession, assistant_id: str) -> dict[str, s
return {binding.capability: binding.model_resource_id for binding in bindings}
async def _sync_tool_bindings(
session: AsyncSession, assistant_id: str, assistant_type: str, tool_ids: list[str]
) -> None:
requested = list(dict.fromkeys(tool_ids)) if assistant_type == "prompt" else []
if requested:
tools = (
await session.execute(select(Tool).where(Tool.id.in_(requested)))
).scalars().all()
found = {tool.id for tool in tools if tool.status == "active"}
missing = [tool_id for tool_id in requested if tool_id not in found]
if missing:
raise HTTPException(400, f"工具不存在或未启用: {', '.join(missing)}")
existing = (
await session.execute(
select(AssistantToolBinding).where(
AssistantToolBinding.assistant_id == assistant_id
)
)
).scalars().all()
existing_by_id = {binding.tool_id: binding for binding in existing}
for tool_id, binding in existing_by_id.items():
if tool_id not in requested:
await session.delete(binding)
for tool_id in requested:
if tool_id not in existing_by_id:
session.add(AssistantToolBinding(assistant_id=assistant_id, tool_id=tool_id))
async def _tool_ids(session: AsyncSession, assistant_id: str) -> list[str]:
rows = (
await session.execute(
select(AssistantToolBinding.tool_id)
.where(AssistantToolBinding.assistant_id == assistant_id)
.order_by(AssistantToolBinding.created_at)
)
).scalars().all()
return list(rows)
async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
return AssistantOut(
id=assistant.id,
@@ -95,6 +141,7 @@ async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
vision_model_resource_id=assistant.vision_model_resource_id,
model_resource_ids=await _resource_ids(session, assistant.id),
knowledge_base_id=assistant.knowledge_base_id,
tool_ids=await _tool_ids(session, assistant.id),
prompt=assistant.prompt,
api_url=assistant.api_url,
api_key=mask(assistant.api_key),
@@ -120,10 +167,12 @@ async def create_assistant(
await _validate_vision_model(session, body)
data = body.model_dump()
resource_ids = data.pop("model_resource_ids")
tool_ids = data.pop("tool_ids")
assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data)
session.add(assistant)
await session.flush()
await _sync_bindings(session, assistant.id, resource_ids)
await _sync_tool_bindings(session, assistant.id, assistant.type, tool_ids)
await session.commit()
await session.refresh(assistant)
return await _to_out(session, assistant)
@@ -165,6 +214,9 @@ async def duplicate_assistant(
session.add(assistant)
await session.flush()
await _sync_bindings(session, assistant.id, await _resource_ids(session, source.id))
await _sync_tool_bindings(
session, assistant.id, assistant.type, await _tool_ids(session, source.id)
)
await session.commit()
await session.refresh(assistant)
return await _to_out(session, assistant)
@@ -183,10 +235,12 @@ async def update_assistant(
await _validate_vision_model(session, body)
data = body.model_dump()
resource_ids = data.pop("model_resource_ids")
tool_ids = data.pop("tool_ids")
data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key)
for key, value in data.items():
setattr(assistant, key, value)
await _sync_bindings(session, assistant.id, resource_ids)
await _sync_tool_bindings(session, assistant.id, assistant.type, tool_ids)
await session.commit()
await session.refresh(assistant)
return await _to_out(session, assistant)

118
backend/routes/tools.py Normal file
View File

@@ -0,0 +1,118 @@
"""Reusable tool CRUD. Tool execution is intentionally not wired to pipelines yet."""
import uuid
from db.models import AssistantToolBinding, Tool
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException
from schemas import ToolOut, ToolUpsert
from services.auth import require_admin
from services.masking import mask_secrets, merge_secrets
from sqlalchemy import select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession
router = APIRouter(
prefix="/api/tools",
tags=["tools"],
dependencies=[Depends(require_admin)],
)
def _to_out(tool: Tool) -> ToolOut:
return ToolOut(
id=tool.id,
name=tool.name,
function_name=tool.function_name,
type=tool.type,
description=tool.description,
definition=tool.definition,
secrets=mask_secrets(tool.secrets or {}),
status=tool.status,
updated_at=tool.updated_at.isoformat() if tool.updated_at else None,
)
def _payload(body: ToolUpsert, stored_secrets: dict | None = None) -> dict:
definition = body.definition.model_dump()
secrets = (
merge_secrets(body.secrets, stored_secrets or {})
if definition["type"] == "http"
else {}
)
return {
"name": body.name.strip(),
"function_name": body.function_name,
"type": definition["type"],
"description": body.description.strip(),
"definition": definition,
"secrets": secrets,
"status": body.status,
}
async def _commit(session: AsyncSession, tool: Tool) -> ToolOut:
try:
await session.commit()
except IntegrityError as exc:
await session.rollback()
raise HTTPException(409, "工具函数名已存在") from exc
await session.refresh(tool)
return _to_out(tool)
@router.get("", response_model=list[ToolOut])
async def list_tools(session: AsyncSession = Depends(get_session)):
rows = (
await session.execute(select(Tool).order_by(Tool.updated_at.desc()))
).scalars().all()
return [_to_out(tool) for tool in rows]
@router.post("", response_model=ToolOut)
async def create_tool(body: ToolUpsert, session: AsyncSession = Depends(get_session)):
tool = Tool(id=f"tool_{uuid.uuid4().hex[:12]}", **_payload(body))
session.add(tool)
return await _commit(session, tool)
@router.get("/{tool_id}", response_model=ToolOut)
async def get_tool(tool_id: str, session: AsyncSession = Depends(get_session)):
tool = await session.get(Tool, tool_id)
if not tool:
raise HTTPException(404, "工具不存在")
return _to_out(tool)
@router.put("/{tool_id}", response_model=ToolOut)
async def update_tool(
tool_id: str,
body: ToolUpsert,
session: AsyncSession = Depends(get_session),
):
tool = await session.get(Tool, tool_id)
if not tool:
raise HTTPException(404, "工具不存在")
for key, value in _payload(body, tool.secrets or {}).items():
setattr(tool, key, value)
return await _commit(session, tool)
@router.delete("/{tool_id}")
async def delete_tool(tool_id: str, session: AsyncSession = Depends(get_session)):
tool = await session.get(Tool, tool_id)
if not tool:
raise HTTPException(404, "工具不存在")
in_use = (
await session.execute(
select(AssistantToolBinding.assistant_id)
.where(AssistantToolBinding.tool_id == tool_id)
.limit(1)
)
).scalar_one_or_none()
if in_use:
raise HTTPException(409, "工具正被助手引用，请先解绑")
await session.delete(tool)
await session.commit()
return {"ok": True}

74
backend/schemas.py
View File

@@ -7,14 +7,18 @@ JSON 用 camelCase(modelId/interfaceType/apiUrl/apiKey),Python 内部用 snake_c
from __future__ import annotations
from typing import Any, Literal
from typing import Annotated, Any, Literal, Union
from pydantic import BaseModel, ConfigDict, Field, model_validator
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from pydantic.alias_generators import to_camel
RuntimeMode = Literal["pipeline", "realtime"]
ModelType = Literal["LLM", "ASR", "TTS", "Realtime", "Embedding", "Agent"]
AssistantType = Literal["prompt", "workflow", "dify", "fastgpt", "opencode"]
ToolType = Literal["end_call", "http"]
ToolStatus = Literal["active", "archived", "draft"]
ToolParameterType = Literal["string", "number", "integer", "boolean", "object", "array"]
ToolParameterLocation = Literal["path", "query", "body", "header"]
# 外部应用类型:其 config.apiKey 是该助手私有密钥,读时打码 / 写时哨兵
EXTERNAL_TYPES = {"dify", "fastgpt", "opencode"}
@@ -57,6 +61,7 @@ class AssistantUpsert(CamelModel):
model_resource_ids: dict[ModelType, str] = Field(default_factory=dict)
knowledge_base_id: str | None = None
tool_ids: list[str] = Field(default_factory=list)
# 瘦类型专属(真列);按 type 取用,无关字段写入时清零
prompt: str = ""
@@ -74,6 +79,8 @@ class AssistantUpsert(CamelModel):
setattr(self, field, "")
if "graph" not in allowed:
self.graph = {}
if self.type != "prompt":
self.tool_ids = []
# 外部托管大脑只能 cascade,拦住不兼容的 realtime
if self.runtime_mode == "realtime" and self.type not in REALTIME_CAPABLE_TYPES:
raise ValueError(f"类型 {self.type} 不支持 realtime 运行模式")
@@ -85,6 +92,69 @@ class AssistantOut(AssistantUpsert):
updated_at: str | None = None
# ---------- 可复用工具 ----------
class ToolParameter(CamelModel):
name: str
type: ToolParameterType = "string"
location: ToolParameterLocation = "body"
description: str = ""
required: bool = True
class EndCallToolConfig(CamelModel):
message_type: Literal["none", "custom"] = "none"
custom_message: str = ""
capture_reason: bool = True
class HttpToolConfig(CamelModel):
method: Literal["GET", "POST", "PUT", "PATCH", "DELETE"] = "GET"
url: str
timeout_seconds: int = Field(default=15, ge=1, le=120)
headers: dict[str, str] = Field(default_factory=dict)
parameters: list[ToolParameter] = Field(default_factory=list)
body: dict[str, Any] = Field(default_factory=dict)
@field_validator("url")
@classmethod
def validate_url(cls, value: str) -> str:
if not value.startswith(("http://", "https://")):
raise ValueError("HTTP 工具 URL 必须使用 http:// 或 https://")
return value
class EndCallToolDefinition(CamelModel):
schema_version: int = 1
type: Literal["end_call"] = "end_call"
config: EndCallToolConfig = Field(default_factory=EndCallToolConfig)
class HttpToolDefinition(CamelModel):
schema_version: int = 1
type: Literal["http"] = "http"
config: HttpToolConfig
ToolDefinition = Annotated[
Union[EndCallToolDefinition, HttpToolDefinition], Field(discriminator="type")
]
class ToolUpsert(CamelModel):
name: str = Field(min_length=1, max_length=128)
function_name: str = Field(pattern=r"^[a-z][a-z0-9_]{0,63}$")
description: str = Field(default="", max_length=2048)
definition: ToolDefinition
secrets: dict[str, Any] = Field(default_factory=dict)
status: ToolStatus = "active"
class ToolOut(ToolUpsert):
id: str
type: ToolType
updated_at: str | None = None
# ---------- 知识库 ----------
class KnowledgeBaseUpsert(CamelModel):
name: str