wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor backend to support interface-definition driven model resources

Browse Source 
- Introduce a new model structure for managing interface definitions and model resources, enhancing the backend's capability to handle various service integrations.
- Update the Makefile to reflect changes in database seeding and resource management commands.
- Remove the deprecated credentials management routes and replace them with a unified model registry API.
- Modify existing routes and schemas to align with the new model structure, ensuring seamless integration with the frontend.
- Enhance database seeding scripts to populate new model resources and their configurations.
- Update README documentation to reflect the new architecture and usage instructions for model resources and interface definitions.
This commit is contained in:
Xin Wang
2026-06-14 19:36:12 +08:00
parent e25dfd4003
commit 90e3e8a0c0
32 changed files with 2577 additions and 1765 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
backend/README.md
View File

@@ -25,7 +25,7 @@ pipecat 把"管线"和"输出方式"解耦:同一条 `STT→LLM→TTS` 管线可
```
ai-video-backend/
├── app.py # FastAPI 入口,挂路由 + CORS
├── config.py # 读 .env,所有 provider 接入点
├── config.py # 读 .env,模型接口环境变量兜底
├── models.py # AssistantConfig(对齐前端 AssistantForm)
├── routes/ # 一个文件一组端点(对齐 dograh routes/)
│ ├── health.py
@@ -33,7 +33,7 @@ ai-video-backend/
│ └── voice_ws.py # WS 裸音频流
├── services/
│ └── pipecat/ # 引擎(对齐 dograh services/pipecat/)
│ ├── service_factory.py # 建 STT/LLM/TTS(加 provider 在此)
│ ├── service_factory.py # 建 STT/LLM/TTS(按 interface_type 分发)
│ ├── transports.py # transport 工厂(加输出方式在此)
│ └── pipeline.py # 管线拼装与运行(transport 无关)
├── Dockerfile
@@ -57,15 +57,33 @@ ai-video-backend/
### 讯飞 ASR / TTS / SuperTTS
讯飞继续复用 `ProviderCredential` 的现有字段，不增加专属列：
讯飞鉴权直接存入对应 `ModelResource.secrets`，接口参数存入 `ModelResource.values`：
- `interface_type`: `xfyun`
- `api_url`: 讯飞 WebSocket URL（`https://` 会自动转为 `wss://`）
- `api_key`: `{"appId":"...","apiKey":"...","apiSecret":"..."}`
- ASR `model_id`: `iat`
- 普通 TTS `model_id`: `tts`
- 超拟人 TTS `model_id`: `supertts`（包含 `/private/` 的 URL 也会自动识别）
- TTS `voice`: 讯飞音色 ID；`speed=1.0` 对应讯飞正常语速 `50`
- 普通语音识别：`interface_type=xfyun-asr`
- 普通语音合成：`interface_type=xfyun-tts`
- 超拟人语音合成：`interface_type=xfyun-super-tts`
- `values.apiUrl` 保存讯飞 WebSocket URL，音色、语速等可选参数也放在 `values`
- `secrets` 分别保存 `appId`、`apiKey`、`apiSecret`
## 接口定义驱动的模型注册表
LLM、ASR、TTS、Embedding、Realtime 使用同一套两层结构：
```text
assistant_model_bindings -> model_resources -> interface_definitions
```
- `interface_definitions`: 定义具体接入协议、能力和动态表单字段。
- `model_resources`: 每条资源自带 `values/secrets`，不复用供应商账号。
- `assistant_model_bindings`: 助手按能力选择模型资源。
`interface_type` 是具体协议，例如 `xfyun-asr`、`xfyun-tts`、
`xfyun-super-tts`，后端严格按它选择服务实现，不根据模型 ID 或 URL 猜测。
API：
- `/api/interface-definitions`: 前端读取字段定义并动态生成 Dialog。
- `/api/model-resources`: 统一模型资源 CRUD，敏感字段逐项打码。
## 本地运行(用 uv,Python 3.12)

7
backend/app.py
View File

@@ -5,7 +5,8 @@
路由分组(对齐 dograh 的 routes/ 结构):
/health 健康检查
/api/assistants 助手 CRUD
/api/credentials 模型凭证 CRUD(key 打码)
/api/interface-definitions 接口定义
/api/model-resources 模型资源 CRUD
/ws/voice WebRTC 输出(浏览器)
/ws/stream WS 输出(裸音频流)
"""
@@ -20,9 +21,9 @@ from fastapi.middleware.cors import CORSMiddleware
from routes import (
assistants,
credentials,
health,
knowledge_bases,
model_registry,
voice_webrtc,
voice_ws,
)
@@ -46,8 +47,8 @@ app.add_middleware(
app.include_router(health.router)
app.include_router(assistants.router)
app.include_router(credentials.router)
app.include_router(knowledge_bases.router)
app.include_router(model_registry.router)
app.include_router(voice_webrtc.router)
app.include_router(voice_ws.router)

2
backend/config.py
View File

@@ -1,4 +1,4 @@
"""集中读取环境变量。所有 provider 的接入点都在这里,改栈只改 .env。"""
"""集中读取环境变量。所有模型接口的环境变量兜底都在这里。"""
import os

101
backend/db/models.py
View File

@@ -1,15 +1,16 @@
"""数据表定义(SQLAlchemy 2.0)。
两张表,职责分离(见设计):
- ProviderCredential:模型凭证(key 明文存,同 dograh,靠 DB 访问控制兜底;读时打码)
- Assistant:助手配置,**只存模型/音色的"选项名",不嵌 key**
模型注册表由接口定义驱动:
- InterfaceDefinition:具体接入协议及其动态表单字段
- ModelResource:模型配置与鉴权值
- AssistantModelBinding:助手按能力选择模型资源
助手运行时再用 kind 去 ProviderCredential 取真 key(services/config_resolver.py)。
"""
from datetime import datetime
from sqlalchemy import JSON, Boolean, DateTime, Float, ForeignKey, String, func
from sqlalchemy import JSON, Boolean, DateTime, ForeignKey, String, func
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
@@ -17,22 +18,39 @@ class Base(DeclarativeBase):
pass
class ProviderCredential(Base):
"""模型资源凭证。字段对齐前端 ComponentsModelsPage 的 ModelResource。"""
class InterfaceDefinition(Base):
"""具体接入协议，例如 xfyun-tts 与 xfyun-super-tts。"""
__tablename__ = "provider_credentials"
__tablename__ = "interface_definitions"
id: Mapped[str] = mapped_column(String(40), primary_key=True) # model_xxx
name: Mapped[str] = mapped_column(String(128), default="") # 资源名称,如 "DeepSeek-V3"
model_id: Mapped[str] = mapped_column(String(128), default="") # 模型ID,如 "deepseek-chat"
type: Mapped[str] = mapped_column(String(16), index=True) # LLM|ASR|TTS|Realtime|Embedding
interface_type: Mapped[str] = mapped_column(String(32), default="openai") # openai|xfyun|dashscope|gemini
api_url: Mapped[str] = mapped_column(String(512), default="")
api_key: Mapped[str] = mapped_column(String(512), default="") # 明文
voice: Mapped[str] = mapped_column(String(128), default="") # TTS 音色
speed: Mapped[float] = mapped_column(Float, default=1.0) # TTS 语速
language: Mapped[str] = mapped_column(String(32), default="") # ASR 语言
# 同一 type 下的默认凭证(后端解析用;前端 ModelResource 无此字段,留作可选)
interface_type: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(128))
capability: Mapped[str] = mapped_column(String(16), index=True)
field_schema: Mapped[dict] = mapped_column(JSONB, default=dict)
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
version: Mapped[int] = mapped_column(default=1)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
class ModelResource(Base):
"""统一模型资源：接口类型决定能力、鉴权字段和调用参数。"""
__tablename__ = "model_resources"
id: Mapped[str] = mapped_column(String(40), primary_key=True)
name: Mapped[str] = mapped_column(String(128), default="")
capability: Mapped[str] = mapped_column(String(16), index=True)
interface_type: Mapped[str] = mapped_column(
String(64),
ForeignKey("interface_definitions.interface_type", ondelete="RESTRICT"),
index=True,
)
values: Mapped[dict] = mapped_column(JSONB, default=dict)
secrets: Mapped[dict] = mapped_column(JSONB, default=dict)
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
@@ -41,7 +59,7 @@ class ProviderCredential(Base):
class KnowledgeBase(Base):
"""知识库注册表。本身引用一个 Embedding 凭证(用哪个向量模型)。
"""知识库注册表。本身引用一个 Embedding 模型资源。
文档/分块(pgvector)是 KB 内部实现,这里先不展开;助手侧只认 knowledge_base_id。
"""
@@ -51,10 +69,9 @@ class KnowledgeBase(Base):
id: Mapped[str] = mapped_column(String(40), primary_key=True) # kb_xxx
name: Mapped[str] = mapped_column(String(128))
description: Mapped[str] = mapped_column(String(2048), default="")
# 该 KB 用哪个向量模型;凭证被删则置空
embedding_credential_id: Mapped[str | None] = mapped_column(
embedding_model_resource_id: Mapped[str | None] = mapped_column(
String(40),
ForeignKey("provider_credentials.id", ondelete="SET NULL"),
ForeignKey("model_resources.id", ondelete="SET NULL"),
nullable=True,
)
status: Mapped[str] = mapped_column(String(16), default="active") # active|archived
@@ -80,19 +97,6 @@ class Assistant(Base):
greeting: Mapped[str] = mapped_column(String(2048), default="")
enable_interrupt: Mapped[bool] = mapped_column(Boolean, default=True)
# ---- 引用"注册好的资源":凭证被删 → SET NULL(resolver 有默认/.env 兜底) ----
llm_credential_id: Mapped[str | None] = mapped_column(
String(40), ForeignKey("provider_credentials.id", ondelete="SET NULL"), nullable=True
)
asr_credential_id: Mapped[str | None] = mapped_column(
String(40), ForeignKey("provider_credentials.id", ondelete="SET NULL"), nullable=True
)
tts_credential_id: Mapped[str | None] = mapped_column(
String(40), ForeignKey("provider_credentials.id", ondelete="SET NULL"), nullable=True
)
realtime_credential_id: Mapped[str | None] = mapped_column(
String(40), ForeignKey("provider_credentials.id", ondelete="SET NULL"), nullable=True
)
# KB 引用:被引用时禁止删 KB(RESTRICT),无默认兜底
knowledge_base_id: Mapped[str | None] = mapped_column(
String(40), ForeignKey("knowledge_bases.id", ondelete="RESTRICT"), nullable=True
@@ -101,7 +105,7 @@ class Assistant(Base):
# ---- 瘦类型专属字段(真列,稀疏:按 type 用其中几列) ----
prompt: Mapped[str] = mapped_column(String(8192), default="") # prompt / opencode
api_url: Mapped[str] = mapped_column(String(512), default="") # dify / fastgpt / opencode
api_key: Mapped[str] = mapped_column(String(512), default="") # dify / fastgpt / opencode(打码/哨兵,同凭证)
api_key: Mapped[str] = mapped_column(String(512), default="") # dify / fastgpt / opencode(打码/哨兵)
app_id: Mapped[str] = mapped_column(String(128), default="") # fastgpt
# workflow 专属:图(nodes/edges)。要版本化时再迁出到 assistant_workflow 表
graph: Mapped[dict] = mapped_column(JSON, default=dict)
@@ -110,3 +114,26 @@ class Assistant(Base):
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)
class AssistantModelBinding(Base):
"""助手按能力绑定统一模型资源；config 可覆盖资源默认 options。"""
__tablename__ = "assistant_model_bindings"
assistant_id: Mapped[str] = mapped_column(
String(40),
ForeignKey("assistants.id", ondelete="CASCADE"),
primary_key=True,
)
capability: Mapped[str] = mapped_column(String(16), primary_key=True)
model_resource_id: Mapped[str] = mapped_column(
String(40),
ForeignKey("model_resources.id", ondelete="RESTRICT"),
index=True,
)
config: Mapped[dict] = mapped_column(JSONB, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), server_default=func.now(), onupdate=func.now()
)

65
backend/db/seed_assistants.sql
View File

@@ -1,51 +1,42 @@
-- 知识库 + 助手种子数据(依赖 seed_credentials.sql 先灌入 model_xxx 凭证)。
--
-- 用法(从仓库根目录):
-- docker compose exec -T postgres psql -U postgres -d postgres < backend/db/seed_assistants.sql
-- 或:make db-seed-assistants(凭证已就绪时);make db-seed 会按顺序全灌。
--
-- 说明:
-- * id 固定(kb_001 / asst_001..005)+ ON CONFLICT 幂等,可重复执行。
-- * 引用 seed_credentials 的 model_001(LLM)/003(ASR)/005(TTS)/010(Embedding)。
-- * 宽表 STI:瘦类型用真列(prompt/api_url/api_key/app_id),workflow 用 graph 列。
-- * api_key 在库里明文(读取走 API 才打码),这里填示例占位。
-- 知识库 + 助手种子数据。依赖 seed_model_resources.sql。
-- 知识库(引用 Embedding 凭证 model_010)
INSERT INTO knowledge_bases (id, name, description, embedding_credential_id, status)
INSERT INTO knowledge_bases
(id, name, description, embedding_model_resource_id, status)
VALUES
('kb_001', '政务政策知识库', '政策解读 / 办事指南示例库', 'model_010', 'active')
('kb_001', '政务政策知识库', '政策解读 / 办事指南示例库', 'model_003', 'active')
ON CONFLICT (id) DO NOTHING;
-- 助手(一种类型一条)
INSERT INTO assistants (
id, name, type, runtime_mode, greeting, enable_interrupt,
llm_credential_id, asr_credential_id, tts_credential_id,
realtime_credential_id, knowledge_base_id,
prompt, api_url, api_key, app_id, graph
knowledge_base_id, prompt, api_url, api_key, app_id, graph
) VALUES
-- 提示词:llm/asr/tts + 知识库,prompt 真列
('asst_001', '政务咨询助手', 'prompt', 'pipeline', '您好，我是政务助手，请问有什么可以帮您？', TRUE,
'model_001', 'model_003', 'model_005', NULL, 'kb_001',
'你是一名专业的政务咨询助手，回答准确、简洁，不编造政策内容。', '', '', '', '{}'),
-- 工作流:asr/tts + graph 列(最小图)
'kb_001', '你是一名专业的政务咨询助手，回答准确、简洁，不编造政策内容。', '', '', '', '{}'),
('asst_002', '热线工单助手', 'workflow', 'pipeline', '', TRUE,
NULL, 'model_003', 'model_005', NULL, NULL,
'', '', '', '',
NULL, '', '', '', '',
'{"nodes":[{"id":"1","type":"startCall","position":{"x":0,"y":0},"data":{"name":"开场","prompt":"你好，请问需要办理什么业务？"}}],"edges":[]}'),
-- Dify:asr/tts + api_url/api_key
('asst_003', 'Dify 客服助手', 'dify', 'pipeline', '', TRUE,
NULL, 'model_003', 'model_005', NULL, NULL,
'', 'https://api.dify.ai/v1', 'app-dify-demo-key', '', '{}'),
-- FastGPT:asr/tts + app_id/api_url/api_key
NULL, '', 'https://api.dify.ai/v1', 'app-dify-demo-key', '', '{}'),
('asst_004', 'FastGPT 售后助手', 'fastgpt', 'pipeline', '', TRUE,
NULL, 'model_003', 'model_005', NULL, NULL,
'', 'https://api.fastgpt.in/api/v1/chat/completions', 'fastgpt-demo-key', 'app-fastgpt-001', '{}'),
-- OpenCode:asr/tts + prompt/api_url/api_key
NULL, '', 'https://api.fastgpt.in/api/v1/chat/completions', 'fastgpt-demo-key', 'app-fastgpt-001', '{}'),
('asst_005', 'OpenCode 代码助手', 'opencode', 'pipeline', '', TRUE,
NULL, 'model_003', 'model_005', NULL, NULL,
'你是一个代码助手的语音界面，用简洁口语回答工程问题。', 'http://localhost:4096', 'opencode-demo-key', '', '{}')
NULL, '你是一个代码助手的语音界面，用简洁口语回答工程问题。', 'http://localhost:4096', 'opencode-demo-key', '', '{}')
ON CONFLICT (id) DO NOTHING;
INSERT INTO assistant_model_bindings
(assistant_id, capability, model_resource_id, config)
VALUES
('asst_001', 'LLM', 'model_001', '{}'),
('asst_001', 'ASR', 'model_002', '{}'),
('asst_001', 'TTS', 'model_004', '{}'),
('asst_002', 'ASR', 'model_002', '{}'),
('asst_002', 'TTS', 'model_004', '{}'),
('asst_003', 'ASR', 'model_002', '{}'),
('asst_003', 'TTS', 'model_004', '{}'),
('asst_004', 'ASR', 'model_002', '{}'),
('asst_004', 'TTS', 'model_004', '{}'),
('asst_005', 'ASR', 'model_002', '{}'),
('asst_005', 'TTS', 'model_004', '{}')
ON CONFLICT (assistant_id, capability) DO UPDATE SET
model_resource_id = EXCLUDED.model_resource_id,
updated_at = now();

38
backend/db/seed_credentials.sql
View File

@@ -1,38 +0,0 @@
-- 模型凭证种子数据(对应前端原 mockModels 的 12 条)。
--
-- 用法(从仓库根目录):
-- docker compose exec -T postgres psql -U postgres -d postgres < backend/db/seed_credentials.sql
--
-- 说明:
-- * id 固定为 model_001..012,配合 ON CONFLICT 做幂等更新。
-- * api_key 在库里是明文(读取走 API 时才打码),这里填的是占位示例 key。
-- * 每种 type 选第一条置为默认(is_default),供后端 config_resolver 解析使用。
-- * TTS 使用 voice/speed;ASR 使用 language;其他类型保持空值/default。
INSERT INTO provider_credentials
(id, name, model_id, type, interface_type, api_url, api_key, voice, speed, language, is_default)
VALUES
('model_001', 'DeepSeek-Chat', 'deepseek-chat', 'LLM', 'openai', 'https://api.deepseek.com/v1', 'sk-230701ff1b6143ecbf322b3170606016', '', 1.0, '', TRUE),
('model_002', 'SiliconFlow-TeleSpeechASR', 'TeleAI/TeleSpeechASR', 'ASR', 'openai', 'https://api.siliconflow.cn/v1', 'sk-uudpgflahqqjbofhgcbwjjefgwhvwwmxgeyehcueqlemwavq', '', 1.0, 'zh', FALSE),
('model_003', 'SiliconFlow-Qwen3-Embedding-4B', 'Qwen/Qwen3-Embedding-4B', 'Embedding', 'openai', 'https://api.siliconflow.cn/v1', 'sk-uudpgflahqqjbofhgcbwjjefgwhvwwmxgeyehcueqlemwavq', '', 1.0, '', TRUE),
('model_004', 'SiliconFlow-CosyVoice2-0.5B', 'FunAudioLLM/CosyVoice2-0.5B', 'TTS', 'openai', 'https://api.siliconflow.cn/v1', 'sk-uudpgflahqqjbofhgcbwjjefgwhvwwmxgeyehcueqlemwavq', 'FunAudioLLM/CosyVoice2-0.5B:anna', 1.0, '', FALSE),
('model_005', 'Qwen-Max', 'qwen-max', 'LLM', 'openai', 'https://dashscope.aliyuncs.com/compatible-mode/v1', 'sk-qwen-4d8e2a6f0c', '', 1.0, '', FALSE),
('model_006', '讯飞语音识别', 'iat', 'ASR', 'xfyun', 'https://iat-api.xfyun.cn/v2/iat', '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', '', 1.0, 'zh', TRUE),
('model_007', 'Paraformer 识别', 'paraformer-realtime-v2', 'ASR', 'dashscope', 'https://dashscope.aliyuncs.com/api/v1/services/audio/asr', 'sk-paraformer-2e4f6a', '', 1.0, 'zh', FALSE),
('model_008', '讯飞语音合成', 'tts', 'TTS', 'xfyun', 'https://tts-api.xfyun.cn/v2/tts', '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', 'xiaoyan', 1.0, '', TRUE),
('model_009', 'CosyVoice 合成', 'cosyvoice-v1', 'TTS', 'dashscope', 'https://dashscope.aliyuncs.com/api/v1/services/audio/tts', 'sk-cosyvoice-1a3c5e', 'longxiaochun', 1.0, '', FALSE),
('model_010', 'GPT Realtime', 'gpt-4o-realtime-preview', 'Realtime', 'openai', 'https://api.openai.com/v1/realtime', 'sk-realtime-3b5d7f9a1c', '', 1.0, '', TRUE),
('model_011', 'Gemini Live', 'gemini-2.0-flash-live', 'Realtime', 'gemini', 'https://generativelanguage.googleapis.com/v1beta', 'gm-live-5e7a9c1b3d', '', 1.0, '', FALSE),
('model_012', 'text-embedding-3', 'text-embedding-3-small', 'Embedding', 'openai', 'https://api.openai.com/v1/embeddings', 'sk-embed-0c2e4a6f8b', '', 1.0, '', FALSE)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
model_id = EXCLUDED.model_id,
type = EXCLUDED.type,
interface_type = EXCLUDED.interface_type,
api_url = EXCLUDED.api_url,
api_key = EXCLUDED.api_key,
voice = EXCLUDED.voice,
speed = EXCLUDED.speed,
language = EXCLUDED.language,
is_default = EXCLUDED.is_default,
updated_at = now();

48
backend/db/seed_model_resources.sql Normal file
View File

@@ -0,0 +1,48 @@
-- 模型资源种子数据。依赖应用启动时写入 interface_definitions。
INSERT INTO model_resources
(id, name, capability, interface_type, values, secrets, enabled, is_default)
VALUES
('model_001', 'DeepSeek-Chat', 'LLM', 'openai-llm',
'{"modelId":"deepseek-chat","apiUrl":"https://api.deepseek.com/v1","temperature":0.7}',
'{"apiKey":"replace-me"}', TRUE, TRUE),
('model_002', 'SiliconFlow-TeleSpeechASR', 'ASR', 'openai-asr',
'{"modelId":"TeleAI/TeleSpeechASR","apiUrl":"https://api.siliconflow.cn/v1","language":"zh"}',
'{"apiKey":"replace-me"}', TRUE, FALSE),
('model_003', 'SiliconFlow-Qwen3-Embedding-4B', 'Embedding', 'openai-embedding',
'{"modelId":"Qwen/Qwen3-Embedding-4B","apiUrl":"https://api.siliconflow.cn/v1"}',
'{"apiKey":"replace-me"}', TRUE, TRUE),
('model_004', 'SiliconFlow-CosyVoice2-0.5B', 'TTS', 'openai-tts',
'{"modelId":"FunAudioLLM/CosyVoice2-0.5B","apiUrl":"https://api.siliconflow.cn/v1","voice":"FunAudioLLM/CosyVoice2-0.5B:anna","speed":1.0,"sourceSampleRate":24000}',
'{"apiKey":"replace-me"}', TRUE, FALSE),
'{"apiKey":"replace-me"}', TRUE, FALSE),
('model_005', '讯飞语音识别', 'ASR', 'xfyun-asr',
'{"apiUrl":"https://iat-api.xfyun.cn/v2/iat","language":"zh_cn","domain":"iat","accent":"mandarin","dynamicCorrection":false,"frameSize":1280}',
'{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE),
('model_006', 'Paraformer 识别', 'ASR', 'dashscope-asr',
'{"modelId":"paraformer-realtime-v2","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/asr","language":"zh"}',
'{"apiKey":"replace-me"}', TRUE, FALSE),
('model_007', '讯飞语音合成', 'TTS', 'xfyun-tts',
'{"apiUrl":"https://tts-api.xfyun.cn/v2/tts","voice":"xiaoyan","speed":50,"volume":50,"pitch":50,"sourceSampleRate":16000}',
'{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE),
('model_008', 'CosyVoice 合成', 'TTS', 'dashscope-tts',
'{"modelId":"cosyvoice-v1","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/tts","voice":"longxiaochun"}',
'{"apiKey":"replace-me"}', TRUE, FALSE),
('model_009', 'GPT Realtime', 'Realtime', 'openai-realtime',
'{"modelId":"gpt-4o-realtime-preview","apiUrl":"https://api.openai.com/v1/realtime"}',
'{"apiKey":"replace-me"}', TRUE, TRUE),
('model_010', 'Gemini Live', 'Realtime', 'gemini-realtime',
'{"modelId":"gemini-2.0-flash-live","apiUrl":"https://generativelanguage.googleapis.com/v1beta"}',
'{"apiKey":"replace-me"}', TRUE, FALSE),
('model_011', 'text-embedding-3', 'Embedding', 'openai-embedding',
'{"modelId":"text-embedding-3-small","apiUrl":"https://api.openai.com/v1/embeddings"}',
'{"apiKey":"replace-me"}', TRUE, FALSE)
ON CONFLICT (id) DO UPDATE SET
name = EXCLUDED.name,
capability = EXCLUDED.capability,
interface_type = EXCLUDED.interface_type,
values = EXCLUDED.values,
secrets = EXCLUDED.secrets,
enabled = EXCLUDED.enabled,
is_default = EXCLUDED.is_default,
updated_at = now();

34
backend/db/session.py
View File

@@ -6,9 +6,11 @@
"""
from collections.abc import AsyncGenerator
import json
import config
from db.models import Base
from services.interface_catalog import INTERFACE_DEFINITIONS
from sqlalchemy import text
from sqlalchemy.ext.asyncio import (
AsyncSession,
@@ -28,22 +30,26 @@ async def get_session() -> AsyncGenerator[AsyncSession, None]:
async def init_db() -> None:
async with engine.begin() as conn:
await conn.run_sync(Base.metadata.create_all)
# MVP 兼容迁移:create_all 不会给已存在的表补列。
await conn.execute(
text(
"ALTER TABLE provider_credentials "
"ADD COLUMN IF NOT EXISTS voice VARCHAR(128) NOT NULL DEFAULT ''"
"ALTER TABLE interface_definitions "
"ALTER COLUMN field_schema TYPE JSONB USING field_schema::jsonb"
)
)
await conn.execute(
text(
"ALTER TABLE provider_credentials "
"ADD COLUMN IF NOT EXISTS speed DOUBLE PRECISION NOT NULL DEFAULT 1.0"
for definition in INTERFACE_DEFINITIONS:
await conn.execute(
text(
"INSERT INTO interface_definitions "
"(interface_type, name, capability, field_schema, enabled, version) "
"VALUES (:interface_type, :name, :capability, CAST(:field_schema AS jsonb), TRUE, 1) "
"ON CONFLICT (interface_type) DO UPDATE SET "
"name = EXCLUDED.name, capability = EXCLUDED.capability, "
"field_schema = EXCLUDED.field_schema, enabled = TRUE, updated_at = now()"
),
{
"interface_type": definition["interface_type"],
"name": definition["name"],
"capability": definition["capability"],
"field_schema": json.dumps({"fields": definition["fields"]}),
},
)
)
await conn.execute(
text(
"ALTER TABLE provider_credentials "
"ADD COLUMN IF NOT EXISTS language VARCHAR(32) NOT NULL DEFAULT ''"
)
)

11
backend/models.py
View File

@@ -31,8 +31,15 @@ class AssistantConfig(BaseModel):
stt_language: str = ""
tts_speed: float = 1.0
realtimeModel: str = ""
stt_interface_type: str = "openai"
tts_interface_type: str = "openai"
llm_interface_type: str = "openai-llm"
stt_interface_type: str = "openai-asr"
tts_interface_type: str = "openai-tts"
llm_values: dict = {}
llm_secrets: dict = {}
stt_values: dict = {}
stt_secrets: dict = {}
tts_values: dict = {}
tts_secrets: dict = {}
enableInterrupt: bool = True

163
backend/routes/assistants.py
View File

@@ -1,12 +1,8 @@
"""助手 CRUD。前端「助手列表 / 创建 / 编辑」对接这里。
模型/KB 以 FK 引用注册表;瘦类型字段直接是真列。外部类型(dify/fastgpt/opencode)的
api_key 是私有密钥,读时打码、写时哨兵(列级,复用 services/masking,与凭证表一致)。
"""
"""Assistant CRUD backed by capability-to-model-resource bindings."""
import uuid
from db.models import Assistant
from db.models import Assistant, AssistantModelBinding, ModelResource
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException
from schemas import AssistantOut, AssistantUpsert
@@ -15,27 +11,62 @@ from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
router = APIRouter(prefix="/api/assistants", tags=["assistants"])
CAPABILITIES = ("LLM", "ASR", "TTS", "Realtime", "Embedding")
def _to_out(a: Assistant) -> AssistantOut:
async def _sync_bindings(
session: AsyncSession, assistant_id: str, resource_ids: dict[str, str]
) -> None:
for capability in CAPABILITIES:
resource_id = resource_ids.get(capability)
binding = await session.get(AssistantModelBinding, (assistant_id, capability))
if not resource_id:
if binding:
await session.delete(binding)
continue
resource = await session.get(ModelResource, resource_id)
if not resource or resource.capability != capability:
raise HTTPException(400, f"{capability} 绑定必须引用同能力的模型资源")
if binding:
binding.model_resource_id = resource_id
else:
session.add(
AssistantModelBinding(
assistant_id=assistant_id,
capability=capability,
model_resource_id=resource_id,
config={},
)
)
async def _resource_ids(session: AsyncSession, assistant_id: str) -> dict[str, str]:
bindings = (
await session.execute(
select(AssistantModelBinding).where(
AssistantModelBinding.assistant_id == assistant_id
)
)
).scalars().all()
return {binding.capability: binding.model_resource_id for binding in bindings}
async def _to_out(session: AsyncSession, assistant: Assistant) -> AssistantOut:
return AssistantOut(
id=a.id,
name=a.name,
type=a.type, # type: ignore[arg-type]
runtime_mode=a.runtime_mode, # type: ignore[arg-type]
greeting=a.greeting,
enable_interrupt=a.enable_interrupt,
llm_credential_id=a.llm_credential_id,
asr_credential_id=a.asr_credential_id,
tts_credential_id=a.tts_credential_id,
realtime_credential_id=a.realtime_credential_id,
knowledge_base_id=a.knowledge_base_id,
prompt=a.prompt,
api_url=a.api_url,
api_key=mask(a.api_key), # 仅外部类型有值;空串 mask 仍是空串
app_id=a.app_id,
graph=a.graph or {},
updated_at=a.updated_at.isoformat() if a.updated_at else None,
id=assistant.id,
name=assistant.name,
type=assistant.type, # type: ignore[arg-type]
runtime_mode=assistant.runtime_mode, # type: ignore[arg-type]
greeting=assistant.greeting,
enable_interrupt=assistant.enable_interrupt,
model_resource_ids=await _resource_ids(session, assistant.id),
knowledge_base_id=assistant.knowledge_base_id,
prompt=assistant.prompt,
api_url=assistant.api_url,
api_key=mask(assistant.api_key),
app_id=assistant.app_id,
graph=assistant.graph or {},
updated_at=assistant.updated_at.isoformat() if assistant.updated_at else None,
)
@@ -44,60 +75,61 @@ async def list_assistants(session: AsyncSession = Depends(get_session)):
rows = (
await session.execute(select(Assistant).order_by(Assistant.updated_at.desc()))
).scalars().all()
return [_to_out(a) for a in rows]
return [await _to_out(session, assistant) for assistant in rows]
@router.post("", response_model=AssistantOut)
async def create_assistant(
body: AssistantUpsert, session: AsyncSession = Depends(get_session)
):
a = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **body.model_dump())
session.add(a)
data = body.model_dump()
resource_ids = data.pop("model_resource_ids")
assistant = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **data)
session.add(assistant)
await session.flush()
await _sync_bindings(session, assistant.id, resource_ids)
await session.commit()
await session.refresh(a)
return _to_out(a)
await session.refresh(assistant)
return await _to_out(session, assistant)
@router.get("/{assistant_id}", response_model=AssistantOut)
async def get_assistant(
assistant_id: str, session: AsyncSession = Depends(get_session)
):
a = await session.get(Assistant, assistant_id)
if not a:
assistant = await session.get(Assistant, assistant_id)
if not assistant:
raise HTTPException(404, "助手不存在")
return _to_out(a)
return await _to_out(session, assistant)
@router.post("/{assistant_id}/duplicate", response_model=AssistantOut)
async def duplicate_assistant(
assistant_id: str, session: AsyncSession = Depends(get_session)
):
"""服务端整行复制:含真实 api_key,DB→DB,密钥不经过浏览器,副本可直接用。"""
src = await session.get(Assistant, assistant_id)
if not src:
source = await session.get(Assistant, assistant_id)
if not source:
raise HTTPException(404, "助手不存在")
a = Assistant(
assistant = Assistant(
id=f"asst_{uuid.uuid4().hex[:12]}",
name=f"{src.name} 副本",
type=src.type,
runtime_mode=src.runtime_mode,
greeting=src.greeting,
enable_interrupt=src.enable_interrupt,
llm_credential_id=src.llm_credential_id,
asr_credential_id=src.asr_credential_id,
tts_credential_id=src.tts_credential_id,
realtime_credential_id=src.realtime_credential_id,
knowledge_base_id=src.knowledge_base_id,
prompt=src.prompt,
api_url=src.api_url,
api_key=src.api_key, # 真 key,DB→DB
app_id=src.app_id,
graph=dict(src.graph or {}), # 浅拷贝,避免与源行共享同一 dict
name=f"{source.name} 副本",
type=source.type,
runtime_mode=source.runtime_mode,
greeting=source.greeting,
enable_interrupt=source.enable_interrupt,
knowledge_base_id=source.knowledge_base_id,
prompt=source.prompt,
api_url=source.api_url,
api_key=source.api_key,
app_id=source.app_id,
graph=dict(source.graph or {}),
)
session.add(a)
session.add(assistant)
await session.flush()
await _sync_bindings(session, assistant.id, await _resource_ids(session, source.id))
await session.commit()
await session.refresh(a)
return _to_out(a)
await session.refresh(assistant)
return await _to_out(session, assistant)
@router.put("/{assistant_id}", response_model=AssistantOut)
@@ -106,26 +138,27 @@ async def update_assistant(
body: AssistantUpsert,
session: AsyncSession = Depends(get_session),
):
a = await session.get(Assistant, assistant_id)
if not a:
assistant = await session.get(Assistant, assistant_id)
if not assistant:
raise HTTPException(404, "助手不存在")
data = body.model_dump()
# 写时哨兵(列级):回传打码/空 api_key → 保留旧 key
data["api_key"] = resolve_incoming_key(data["api_key"], a.api_key)
for k, v in data.items():
setattr(a, k, v)
resource_ids = data.pop("model_resource_ids")
data["api_key"] = resolve_incoming_key(data["api_key"], assistant.api_key)
for key, value in data.items():
setattr(assistant, key, value)
await _sync_bindings(session, assistant.id, resource_ids)
await session.commit()
await session.refresh(a)
return _to_out(a)
await session.refresh(assistant)
return await _to_out(session, assistant)
@router.delete("/{assistant_id}")
async def delete_assistant(
assistant_id: str, session: AsyncSession = Depends(get_session)
):
a = await session.get(Assistant, assistant_id)
if not a:
assistant = await session.get(Assistant, assistant_id)
if not assistant:
raise HTTPException(404, "助手不存在")
await session.delete(a)
await session.delete(assistant)
await session.commit()
return {"ok": True}

190
backend/routes/credentials.py
View File

@@ -1,190 +0,0 @@
"""模型资源凭证 CRUD。前端 ComponentsModelsPage 对接这里。
字段对齐前端 ModelResource。读:api_key 打码;写:占位符表示不改(写时哨兵)。
设默认时清掉同 type 其它默认。
"""
import uuid
from db.models import ProviderCredential
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException
from schemas import (
CredentialOut,
CredentialTestRequest,
CredentialTestResult,
CredentialUpsert,
)
from services.credential_tester import test_openai_credential, test_xfyun_credential
from services.masking import mask, resolve_incoming_key
from sqlalchemy import select, update
from sqlalchemy.ext.asyncio import AsyncSession
router = APIRouter(prefix="/api/credentials", tags=["credentials"])
def _to_out(c: ProviderCredential) -> CredentialOut:
return CredentialOut(
id=c.id,
name=c.name,
model_id=c.model_id,
type=c.type,
interface_type=c.interface_type,
api_url=c.api_url,
api_key=mask(c.api_key), # 永远打码
voice=c.voice,
speed=c.speed,
language=c.language,
is_default=c.is_default,
)
async def _clear_other_defaults(session: AsyncSession, type_: str, keep_id: str):
await session.execute(
update(ProviderCredential)
.where(ProviderCredential.type == type_, ProviderCredential.id != keep_id)
.values(is_default=False)
)
@router.get("", response_model=list[CredentialOut])
async def list_credentials(session: AsyncSession = Depends(get_session)):
rows = (
await session.execute(
select(ProviderCredential).order_by(ProviderCredential.type)
)
).scalars().all()
return [_to_out(c) for c in rows]
@router.post("", response_model=CredentialOut)
async def create_credential(
body: CredentialUpsert, session: AsyncSession = Depends(get_session)
):
c = ProviderCredential(
id=f"model_{uuid.uuid4().hex[:12]}",
name=body.name,
model_id=body.model_id,
type=body.type,
interface_type=body.interface_type,
api_url=body.api_url,
api_key=resolve_incoming_key(body.api_key, ""),
voice=body.voice,
speed=body.speed,
language=body.language,
is_default=body.is_default,
)
session.add(c)
if c.is_default:
await _clear_other_defaults(session, c.type, c.id)
await session.commit()
await session.refresh(c)
return _to_out(c)
@router.post("/test", response_model=CredentialTestResult)
async def test_new_credential(body: CredentialTestRequest):
if body.interface_type == "xfyun":
return test_xfyun_credential(body)
if body.interface_type != "openai":
return CredentialTestResult(
ok=False,
message="暂不支持该接口类型",
detail="当前仅支持 OpenAI 兼容接口测试",
)
if not body.api_key:
return CredentialTestResult(
ok=False,
message="缺少 API Key",
detail="测试新配置时需要输入 API Key",
)
return await test_openai_credential(body)
@router.post("/{cred_id}/test", response_model=CredentialTestResult)
async def test_saved_credential(
cred_id: str,
body: CredentialTestRequest,
session: AsyncSession = Depends(get_session),
):
c = await session.get(ProviderCredential, cred_id)
if not c:
raise HTTPException(404, "凭证不存在")
config = body.model_copy(
update={"api_key": resolve_incoming_key(body.api_key, c.api_key)}
)
if config.interface_type == "xfyun":
return test_xfyun_credential(config)
if config.interface_type != "openai":
return CredentialTestResult(
ok=False,
message="暂不支持该接口类型",
detail="当前仅支持 OpenAI 兼容接口测试",
)
return await test_openai_credential(config)
@router.post("/{cred_id}/duplicate", response_model=CredentialOut)
async def duplicate_credential(
cred_id: str, session: AsyncSession = Depends(get_session)
):
"""服务端整行复制:含真实 api_key,DB→DB,密钥不经浏览器。副本不继承默认标记。"""
src = await session.get(ProviderCredential, cred_id)
if not src:
raise HTTPException(404, "凭证不存在")
c = ProviderCredential(
id=f"model_{uuid.uuid4().hex[:12]}",
name=f"{src.name} 副本",
model_id=src.model_id,
type=src.type,
interface_type=src.interface_type,
api_url=src.api_url,
api_key=src.api_key, # 真 key,DB→DB
voice=src.voice,
speed=src.speed,
language=src.language,
is_default=False, # 副本不继承默认,避免抢走源的默认标记
)
session.add(c)
await session.commit()
await session.refresh(c)
return _to_out(c)
@router.put("/{cred_id}", response_model=CredentialOut)
async def update_credential(
cred_id: str,
body: CredentialUpsert,
session: AsyncSession = Depends(get_session),
):
c = await session.get(ProviderCredential, cred_id)
if not c:
raise HTTPException(404, "凭证不存在")
c.name = body.name
c.model_id = body.model_id
c.type = body.type
c.interface_type = body.interface_type
c.api_url = body.api_url
c.voice = body.voice
c.speed = body.speed
c.language = body.language
c.is_default = body.is_default
# 写时哨兵:打码占位符 → 保留旧 key
c.api_key = resolve_incoming_key(body.api_key, c.api_key)
if c.is_default:
await _clear_other_defaults(session, c.type, c.id)
await session.commit()
await session.refresh(c)
return _to_out(c)
@router.delete("/{cred_id}")
async def delete_credential(
cred_id: str, session: AsyncSession = Depends(get_session)
):
c = await session.get(ProviderCredential, cred_id)
if not c:
raise HTTPException(404, "凭证不存在")
await session.delete(c)
await session.commit()
return {"ok": True}

18
backend/routes/knowledge_bases.py
View File

@@ -1,12 +1,12 @@
"""知识库 CRUD。前端助手编辑页的"知识库"下拉对接这里。
KB 自身引用一个 Embedding 凭证(embeddingCredentialId)。被助手引用时禁止删除
KB 自身引用一个 Embedding 模型资源。被助手引用时禁止删除
(DB 层 ON DELETE RESTRICT),这里把外键冲突翻译成 409。
"""
import uuid
from db.models import KnowledgeBase
from db.models import KnowledgeBase, ModelResource
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException
from schemas import KnowledgeBaseOut, KnowledgeBaseUpsert
@@ -17,12 +17,22 @@ from sqlalchemy.ext.asyncio import AsyncSession
router = APIRouter(prefix="/api/knowledge-bases", tags=["knowledge-bases"])
async def _validate_embedding_resource(
session: AsyncSession, resource_id: str | None
) -> None:
if not resource_id:
return
resource = await session.get(ModelResource, resource_id)
if not resource or resource.capability != "Embedding":
raise HTTPException(400, "知识库必须引用 Embedding 模型资源")
def _to_out(kb: KnowledgeBase) -> KnowledgeBaseOut:
return KnowledgeBaseOut(
id=kb.id,
name=kb.name,
description=kb.description,
embedding_credential_id=kb.embedding_credential_id,
embedding_model_resource_id=kb.embedding_model_resource_id,
status=kb.status,
updated_at=kb.updated_at.isoformat() if kb.updated_at else None,
)
@@ -40,6 +50,7 @@ async def list_knowledge_bases(session: AsyncSession = Depends(get_session)):
async def create_knowledge_base(
body: KnowledgeBaseUpsert, session: AsyncSession = Depends(get_session)
):
await _validate_embedding_resource(session, body.embedding_model_resource_id)
kb = KnowledgeBase(id=f"kb_{uuid.uuid4().hex[:12]}", **body.model_dump())
session.add(kb)
await session.commit()
@@ -66,6 +77,7 @@ async def update_knowledge_base(
kb = await session.get(KnowledgeBase, kb_id)
if not kb:
raise HTTPException(404, "知识库不存在")
await _validate_embedding_resource(session, body.embedding_model_resource_id)
for k, v in body.model_dump().items():
setattr(kb, k, v)
await session.commit()

249
backend/routes/model_registry.py Normal file
View File

@@ -0,0 +1,249 @@
"""Interface-definition driven model resource registry APIs."""
import uuid
from db.models import (
AssistantModelBinding,
InterfaceDefinition,
KnowledgeBase,
ModelResource,
)
from db.session import get_session
from fastapi import APIRouter, Depends, HTTPException, Query
from schemas import (
InterfaceDefinitionOut,
ModelResourceOut,
ModelResourceTestResult,
ModelResourceUpsert,
)
from services.interface_catalog import validate_fields
from services.masking import mask_secrets, merge_secrets
from services.model_resource_tester import test_model_resource
from sqlalchemy import delete, select, update
from sqlalchemy.ext.asyncio import AsyncSession
router = APIRouter(prefix="/api", tags=["model-registry"])
def _definition_dict(row: InterfaceDefinition) -> dict:
return {
"interface_type": row.interface_type,
"name": row.name,
"capability": row.capability,
"fields": (row.field_schema or {}).get("fields", []),
}
def _definition_out(row: InterfaceDefinition) -> InterfaceDefinitionOut:
return InterfaceDefinitionOut(
interface_type=row.interface_type,
name=row.name,
capability=row.capability, # type: ignore[arg-type]
field_schema=row.field_schema or {},
enabled=row.enabled,
version=row.version,
)
def _resource_out(row: ModelResource) -> ModelResourceOut:
return ModelResourceOut(
id=row.id,
name=row.name,
capability=row.capability, # type: ignore[arg-type]
interface_type=row.interface_type,
values=row.values or {},
secrets=mask_secrets(row.secrets or {}),
enabled=row.enabled,
is_default=row.is_default,
updated_at=row.updated_at.isoformat() if row.updated_at else None,
)
async def _definition(
session: AsyncSession, interface_type: str
) -> InterfaceDefinition:
row = await session.get(InterfaceDefinition, interface_type)
if not row or not row.enabled:
raise HTTPException(400, f"接口类型不可用: {interface_type}")
return row
async def _validate(
session: AsyncSession,
body: ModelResourceUpsert,
stored_secrets: dict | None = None,
) -> tuple[InterfaceDefinition, dict]:
definition = await _definition(session, body.interface_type)
secrets = merge_secrets(body.secrets, stored_secrets or {})
try:
validate_fields(_definition_dict(definition), body.values, secrets)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return definition, secrets
async def _clear_incompatible_references(
session: AsyncSession, resource: ModelResource, capability: str
) -> None:
if capability == resource.capability:
return
await session.execute(
delete(AssistantModelBinding).where(
AssistantModelBinding.model_resource_id == resource.id
)
)
await session.execute(
update(KnowledgeBase)
.where(KnowledgeBase.embedding_model_resource_id == resource.id)
.values(embedding_model_resource_id=None)
)
@router.get("/interface-definitions", response_model=list[InterfaceDefinitionOut])
async def list_interface_definitions(
capability: str | None = Query(default=None),
session: AsyncSession = Depends(get_session),
):
stmt = select(InterfaceDefinition).where(InterfaceDefinition.enabled.is_(True))
if capability:
stmt = stmt.where(InterfaceDefinition.capability == capability)
rows = (await session.execute(stmt.order_by(InterfaceDefinition.capability))).scalars().all()
return [_definition_out(row) for row in rows]
@router.get("/model-resources", response_model=list[ModelResourceOut])
async def list_model_resources(session: AsyncSession = Depends(get_session)):
rows = (
await session.execute(select(ModelResource).order_by(ModelResource.capability))
).scalars().all()
return [_resource_out(row) for row in rows]
@router.post("/model-resources", response_model=ModelResourceOut)
async def create_model_resource(
body: ModelResourceUpsert, session: AsyncSession = Depends(get_session)
):
definition, secrets = await _validate(session, body)
row = ModelResource(
id=f"model_{uuid.uuid4().hex[:12]}",
name=body.name,
capability=definition.capability,
interface_type=definition.interface_type,
values=body.values,
secrets=secrets,
enabled=body.enabled,
is_default=body.is_default,
)
session.add(row)
if row.is_default:
await session.execute(
update(ModelResource)
.where(ModelResource.capability == row.capability, ModelResource.id != row.id)
.values(is_default=False)
)
await session.commit()
return _resource_out(row)
@router.post("/model-resources/test", response_model=ModelResourceTestResult)
async def test_new_model_resource(
body: ModelResourceUpsert, session: AsyncSession = Depends(get_session)
):
definition, secrets = await _validate(session, body)
return await test_model_resource(
definition.interface_type,
definition.capability,
body.values,
secrets,
)
@router.post(
"/model-resources/{resource_id}/test", response_model=ModelResourceTestResult
)
async def test_saved_model_resource(
resource_id: str,
body: ModelResourceUpsert,
session: AsyncSession = Depends(get_session),
):
row = await session.get(ModelResource, resource_id)
if not row:
raise HTTPException(404, "模型资源不存在")
definition, secrets = await _validate(session, body, row.secrets or {})
return await test_model_resource(
definition.interface_type,
definition.capability,
body.values,
secrets,
)
@router.post("/model-resources/{resource_id}/duplicate", response_model=ModelResourceOut)
async def duplicate_model_resource(
resource_id: str, session: AsyncSession = Depends(get_session)
):
source = await session.get(ModelResource, resource_id)
if not source:
raise HTTPException(404, "模型资源不存在")
row = ModelResource(
id=f"model_{uuid.uuid4().hex[:12]}",
name=f"{source.name} 副本",
capability=source.capability,
interface_type=source.interface_type,
values=dict(source.values or {}),
secrets=dict(source.secrets or {}),
enabled=source.enabled,
is_default=False,
)
session.add(row)
await session.commit()
return _resource_out(row)
@router.put("/model-resources/{resource_id}", response_model=ModelResourceOut)
async def update_model_resource(
resource_id: str,
body: ModelResourceUpsert,
session: AsyncSession = Depends(get_session),
):
row = await session.get(ModelResource, resource_id)
if not row:
raise HTTPException(404, "模型资源不存在")
definition, secrets = await _validate(session, body, row.secrets or {})
await _clear_incompatible_references(session, row, definition.capability)
row.name = body.name
row.capability = definition.capability
row.interface_type = definition.interface_type
row.values = body.values
row.secrets = secrets
row.enabled = body.enabled
row.is_default = body.is_default
if row.is_default:
await session.execute(
update(ModelResource)
.where(ModelResource.capability == row.capability, ModelResource.id != row.id)
.values(is_default=False)
)
await session.commit()
return _resource_out(row)
@router.delete("/model-resources/{resource_id}")
async def delete_model_resource(
resource_id: str, session: AsyncSession = Depends(get_session)
):
row = await session.get(ModelResource, resource_id)
if not row:
raise HTTPException(404, "模型资源不存在")
in_use = (
await session.execute(
select(AssistantModelBinding.assistant_id)
.where(AssistantModelBinding.model_resource_id == resource_id)
.limit(1)
)
).scalar_one_or_none()
if in_use:
raise HTTPException(409, "该模型资源仍被助手引用")
await session.delete(row)
await session.commit()
return {"ok": True}

78
backend/schemas.py
View File

@@ -1,6 +1,6 @@
"""面向前端的请求/响应 DTO。与 DB 模型解耦,**响应里的 key 一律打码**。
凭证 DTO 字段对齐前端 ComponentsModelsPage 的 ModelResource:
模型资源 DTO 字段对齐前端 ComponentsModelsPage 的 ModelResource:
JSON 用 camelCase(modelId/interfaceType/apiUrl/apiKey),Python 内部用 snake_case,
靠 Pydantic alias 自动互转。FastAPI 响应默认 by_alias=True,所以出参也是 camelCase。
"""
@@ -9,12 +9,11 @@ from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, ConfigDict, model_validator
from pydantic import BaseModel, ConfigDict, Field, model_validator
from pydantic.alias_generators import to_camel
RuntimeMode = Literal["pipeline", "realtime"]
ModelType = Literal["LLM", "ASR", "TTS", "Realtime", "Embedding"]
InterfaceType = Literal["openai", "xfyun", "dashscope", "gemini"]
AssistantType = Literal["prompt", "workflow", "dify", "fastgpt", "opencode"]
# 外部应用类型:其 config.apiKey 是该助手私有密钥,读时打码 / 写时哨兵
@@ -49,11 +48,7 @@ class AssistantUpsert(CamelModel):
greeting: str = ""
enable_interrupt: bool = True
# 引用注册资源(FK id;None=未选)
llm_credential_id: str | None = None
asr_credential_id: str | None = None
tts_credential_id: str | None = None
realtime_credential_id: str | None = None
model_resource_ids: dict[ModelType, str] = Field(default_factory=dict)
knowledge_base_id: str | None = None
# 瘦类型专属(真列);按 type 取用,无关字段写入时清零
@@ -62,7 +57,7 @@ class AssistantUpsert(CamelModel):
api_key: str = "" # 写时:占位符/空 → 保留旧(哨兵)
app_id: str = ""
# workflow 专属:图
graph: dict[str, Any] = {}
graph: dict[str, Any] = Field(default_factory=dict)
@model_validator(mode="after")
def _strip_irrelevant_fields(self):
@@ -84,7 +79,7 @@ class AssistantOut(AssistantUpsert):
class KnowledgeBaseUpsert(CamelModel):
name: str
description: str = ""
embedding_credential_id: str | None = None
embedding_model_resource_id: str | None = None
class KnowledgeBaseOut(KnowledgeBaseUpsert):
@@ -93,55 +88,32 @@ class KnowledgeBaseOut(KnowledgeBaseUpsert):
updated_at: str | None = None
# ---------- 模型凭证(对齐前端 ModelResource) ----------
class CredentialUpsert(CamelModel):
name: str = "" # 资源名称
model_id: str = "" # 模型ID
type: ModelType # LLM/ASR/TTS/Realtime/Embedding
interface_type: InterfaceType = "openai" # openai/xfyun/dashscope/gemini
api_url: str = ""
api_key: str = "" # 写时:占位符/空表示不改
voice: str = "" # TTS
speed: float = 1.0 # TTS
language: str = "" # ASR
# ---------- 接口定义驱动的统一模型资源 ----------
class InterfaceDefinitionOut(CamelModel):
interface_type: str
name: str
capability: ModelType
field_schema: dict[str, Any]
enabled: bool
version: int
class ModelResourceUpsert(CamelModel):
name: str
interface_type: str
values: dict[str, Any] = Field(default_factory=dict)
secrets: dict[str, Any] = Field(default_factory=dict)
enabled: bool = True
is_default: bool = False
@model_validator(mode="after")
def _strip_irrelevant_options(self):
if self.type != "TTS":
self.voice = ""
self.speed = 1.0
if self.type != "ASR":
self.language = ""
return self
class CredentialOut(CamelModel):
class ModelResourceOut(ModelResourceUpsert):
id: str
name: str
model_id: str
type: str
interface_type: str
api_url: str
api_key: str # 读时:打码后的值
voice: str
speed: float
language: str
is_default: bool
capability: ModelType
updated_at: str | None = None
class CredentialTestRequest(CamelModel):
model_id: str
type: ModelType
interface_type: InterfaceType = "openai"
api_url: str
api_key: str = ""
voice: str = ""
speed: float = 1.0
language: str = ""
class CredentialTestResult(CamelModel):
class ModelResourceTestResult(CamelModel):
ok: bool
latency_ms: int | None = None
message: str

108
backend/services/config_resolver.py
View File

@@ -1,52 +1,63 @@
"""assistant_id → 运行时配置(把真 key 在服务端组装好)。
浏览器只传 assistant_id;真 key 在这里从 provider_credentials 取出注入。
助手按 FK(*_credential_id)引用凭证;取不到则回退该 type 默认凭证,再回退 .env。
浏览器只传 assistant_id;真 key 在这里从 model_resources 取出注入。
助手按 capability binding 引用资源;取不到则回退该能力默认资源,再回退 .env。
"""
import config
from db.models import Assistant, ProviderCredential
from db.models import Assistant, AssistantModelBinding, ModelResource
from models import AssistantConfig
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
async def _default_credential(
session: AsyncSession, type_: str
) -> ProviderCredential | None:
"""该 type 的默认凭证(is_default 优先,否则按 id 取第一条)。"""
stmt = (
select(ProviderCredential)
.where(ProviderCredential.type == type_)
.order_by(ProviderCredential.is_default.desc(), ProviderCredential.id.asc())
.limit(1)
)
return (await session.execute(stmt)).scalar_one_or_none()
async def _resource_for(
session: AsyncSession,
assistant_id: str,
capability: str,
) -> ModelResource | None:
binding = await session.get(AssistantModelBinding, (assistant_id, capability))
resource_id = binding.model_resource_id if binding else None
resource = await session.get(ModelResource, resource_id) if resource_id else None
if resource and resource.capability != capability:
resource = None
if resource is None:
resource = (
await session.execute(
select(ModelResource)
.where(ModelResource.capability == capability, ModelResource.enabled.is_(True))
.order_by(ModelResource.is_default.desc(), ModelResource.id.asc())
.limit(1)
)
).scalar_one_or_none()
return resource
async def _resolve(
session: AsyncSession, cred_id: str | None, type_: str
) -> ProviderCredential | None:
"""按 FK id 取凭证;id 为空或失效 → 回退该 type 默认。"""
if cred_id:
cred = await session.get(ProviderCredential, cred_id)
if cred:
return cred
return await _default_credential(session, type_)
def _value(resource: ModelResource | None, key: str, default):
if not resource:
return default
value = (resource.values or {}).get(key, default)
return default if value is None else value
def _secret(resource: ModelResource | None, key: str, default: str) -> str:
if not resource:
return default
return str((resource.secrets or {}).get(key) or default)
async def resolve_runtime_config(
session: AsyncSession, assistant_id: str
) -> AssistantConfig:
"""加载助手 + 解析凭证,产出可直接交给管线的运行时配置(含真 key)。"""
"""加载助手 + 解析模型资源,产出可直接交给管线的运行时配置(含真 key)。"""
assistant = await session.get(Assistant, assistant_id)
if assistant is None:
raise ValueError(f"助手不存在: {assistant_id}")
llm = await _resolve(session, assistant.llm_credential_id, "LLM")
stt = await _resolve(session, assistant.asr_credential_id, "ASR")
tts = await _resolve(session, assistant.tts_credential_id, "TTS")
realtime = await _resolve(session, assistant.realtime_credential_id, "Realtime")
llm_resource = await _resource_for(session, assistant.id, "LLM")
stt_resource = await _resource_for(session, assistant.id, "ASR")
tts_resource = await _resource_for(session, assistant.id, "TTS")
realtime_resource = await _resource_for(session, assistant.id, "Realtime")
return AssistantConfig(
name=assistant.name,
@@ -55,21 +66,28 @@ async def resolve_runtime_config(
prompt=assistant.prompt or "你是一个有帮助的助手。",
runtimeMode=assistant.runtime_mode, # type: ignore[arg-type]
enableInterrupt=assistant.enable_interrupt,
# 模型/音色:凭证的模型ID优先
model=(llm.model_id if llm else ""),
asr=(stt.model_id if stt else ""),
tts_model=(tts.model_id if tts else ""),
voice=(tts.voice if tts else ""),
stt_language=(stt.language if stt else ""),
tts_speed=(tts.speed if tts else 1.0),
stt_interface_type=(stt.interface_type if stt else "openai"),
tts_interface_type=(tts.interface_type if tts else "openai"),
realtimeModel=(realtime.model_id if realtime else ""),
# 运行时连接信息(真 key + url):凭证优先,否则 .env 兜底
llm_api_key=(llm.api_key if llm else config.LLM_API_KEY),
llm_base_url=(llm.api_url if llm else config.LLM_BASE_URL),
stt_api_key=(stt.api_key if stt else config.STT_API_KEY),
stt_base_url=(stt.api_url if stt else config.STT_BASE_URL),
tts_api_key=(tts.api_key if tts else config.TTS_API_KEY),
tts_base_url=(tts.api_url if tts else config.TTS_BASE_URL),
# 模型/音色:模型资源中的配置优先
model=str(_value(llm_resource, "modelId", "")),
asr=str(_value(stt_resource, "modelId", "")),
tts_model=str(_value(tts_resource, "modelId", "")),
voice=str(_value(tts_resource, "voice", "")),
stt_language=str(_value(stt_resource, "language", "")),
tts_speed=float(_value(tts_resource, "speed", 1.0)),
llm_interface_type=(llm_resource.interface_type if llm_resource else "openai-llm"),
stt_interface_type=(stt_resource.interface_type if stt_resource else "openai-asr"),
tts_interface_type=(tts_resource.interface_type if tts_resource else "openai-tts"),
realtimeModel=str(_value(realtime_resource, "modelId", "")),
llm_values=(llm_resource.values or {}) if llm_resource else {},
llm_secrets=(llm_resource.secrets or {}) if llm_resource else {},
stt_values=(stt_resource.values or {}) if stt_resource else {},
stt_secrets=(stt_resource.secrets or {}) if stt_resource else {},
tts_values=(tts_resource.values or {}) if tts_resource else {},
tts_secrets=(tts_resource.secrets or {}) if tts_resource else {},
# 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底
llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY),
llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)),
stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY),
stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)),
tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY),
tts_base_url=str(_value(tts_resource, "apiUrl", config.TTS_BASE_URL)),
)

155
backend/services/interface_catalog.py Normal file
View File

@@ -0,0 +1,155 @@
"""Built-in concrete interface definitions used by backend and dynamic forms."""
from __future__ import annotations
from typing import Any
def field(
key: str,
label: str,
*,
group: str = "values",
type_: str = "text",
required: bool = False,
default: Any = None,
options: list[str] | None = None,
) -> dict:
value = {
"key": key,
"label": label,
"group": group,
"type": type_,
"required": required,
}
if default is not None:
value["default"] = default
if options:
value["options"] = options
return value
OPENAI_COMMON = [
field("modelId", "Model ID", required=True),
field("apiUrl", "API URL", type_="url", required=True),
field("apiKey", "API Key", group="secrets", type_="password", required=True),
]
XFYUN_AUTH = [
field("apiUrl", "WebSocket URL", type_="url", required=True),
field("appId", "App ID", group="secrets", type_="password", required=True),
field("apiKey", "API Key", group="secrets", type_="password", required=True),
field("apiSecret", "API Secret", group="secrets", type_="password", required=True),
]
INTERFACE_DEFINITIONS: list[dict] = [
{
"interface_type": "openai-llm",
"name": "OpenAI Compatible LLM",
"capability": "LLM",
"fields": OPENAI_COMMON
+ [field("temperature", "Temperature", type_="number", default=0.7)],
},
{
"interface_type": "openai-asr",
"name": "OpenAI Compatible ASR",
"capability": "ASR",
"fields": OPENAI_COMMON + [field("language", "Language", default="zh")],
},
{
"interface_type": "openai-tts",
"name": "OpenAI Compatible TTS",
"capability": "TTS",
"fields": OPENAI_COMMON
+ [
field("voice", "Voice"),
field("speed", "Speed", type_="number", default=1.0),
field("sourceSampleRate", "Source Sample Rate", type_="number", default=24000),
],
},
{
"interface_type": "openai-embedding",
"name": "OpenAI Compatible Embedding",
"capability": "Embedding",
"fields": OPENAI_COMMON + [field("dimensions", "Dimensions", type_="number")],
},
{
"interface_type": "openai-realtime",
"name": "OpenAI Realtime",
"capability": "Realtime",
"fields": OPENAI_COMMON + [field("voice", "Voice")],
},
{
"interface_type": "xfyun-asr",
"name": "Xfyun Streaming ASR",
"capability": "ASR",
"fields": XFYUN_AUTH
+ [
field("language", "Language", default="zh_cn"),
field("domain", "Domain", default="iat"),
field("accent", "Accent", default="mandarin"),
field("dynamicCorrection", "Dynamic Correction", type_="boolean", default=False),
field("frameSize", "Frame Size", type_="number", default=1280),
],
},
{
"interface_type": "xfyun-tts",
"name": "Xfyun TTS",
"capability": "TTS",
"fields": XFYUN_AUTH
+ [
field("voice", "Voice"),
field("speed", "Speed", type_="number", default=50),
field("volume", "Volume", type_="number", default=50),
field("pitch", "Pitch", type_="number", default=50),
field("sourceSampleRate", "Source Sample Rate", type_="number", default=16000),
],
},
{
"interface_type": "xfyun-super-tts",
"name": "Xfyun Super TTS",
"capability": "TTS",
"fields": XFYUN_AUTH
+ [
field("voice", "Voice"),
field("speed", "Speed", type_="number", default=50),
field("volume", "Volume", type_="number", default=50),
field("pitch", "Pitch", type_="number", default=50),
field("oralLevel", "Oral Level", default="mid"),
field("sourceSampleRate", "Source Sample Rate", type_="number", default=24000),
field("textAggregationMode", "Text Aggregation Mode", default="token"),
],
},
{
"interface_type": "dashscope-llm",
"name": "DashScope LLM",
"capability": "LLM",
"fields": OPENAI_COMMON
+ [field("temperature", "Temperature", type_="number", default=0.7)],
},
{
"interface_type": "dashscope-asr",
"name": "DashScope ASR",
"capability": "ASR",
"fields": OPENAI_COMMON + [field("language", "Language", default="zh")],
},
{
"interface_type": "dashscope-tts",
"name": "DashScope TTS",
"capability": "TTS",
"fields": OPENAI_COMMON + [field("voice", "Voice")],
},
{
"interface_type": "gemini-realtime",
"name": "Gemini Realtime",
"capability": "Realtime",
"fields": OPENAI_COMMON,
},
]
def validate_fields(definition: dict, values: dict, secrets: dict) -> None:
for item in definition["fields"]:
source = secrets if item["group"] == "secrets" else values
value = source.get(item["key"])
if item.get("required") and (value is None or value == ""):
raise ValueError(f"{item['label']} is required")

20
backend/services/masking.py
View File

@@ -25,3 +25,23 @@ def resolve_incoming_key(incoming: str | None, stored: str) -> str:
if incoming is None or incoming == "" or is_masked(incoming):
return stored
return incoming
def mask_secrets(value):
"""Recursively mask every scalar in a model resource secrets object."""
if isinstance(value, dict):
return {key: mask_secrets(item) for key, item in value.items()}
if isinstance(value, list):
return [mask_secrets(item) for item in value]
return mask(str(value)) if value is not None else ""
def merge_secrets(incoming: dict, stored: dict) -> dict:
"""Merge secret fields while treating masked/empty values as keep-existing."""
result = dict(stored or {})
for key, value in (incoming or {}).items():
if isinstance(value, dict) and isinstance(result.get(key), dict):
result[key] = merge_secrets(value, result[key])
elif value not in (None, "") and not is_masked(str(value)):
result[key] = value
return result

114
backend/services/credential_tester.py → backend/services/model_resource_tester.py
View File

@@ -1,4 +1,4 @@
"""OpenAI 兼容模型凭证的最小连通测试。"""
"""Connectivity checks for interface-definition driven model resources."""
from __future__ import annotations
@@ -8,8 +8,8 @@ import wave
import httpx
from schemas import CredentialTestRequest, CredentialTestResult
from services.pipecat.xfyun_config import parse_xfyun_credential
import config
from schemas import ModelResourceTestResult
TEST_TIMEOUT_SECONDS = 10.0
@@ -28,7 +28,7 @@ def _silent_wav() -> bytes:
return buffer.getvalue()
def _error_detail(response: httpx.Response, api_key: str) -> str:
def _error_detail(response: httpx.Response, secrets: dict) -> str:
try:
body = response.json()
detail = (
@@ -39,110 +39,114 @@ def _error_detail(response: httpx.Response, api_key: str) -> str:
except ValueError:
detail = None
text = str(detail or response.text or response.reason_phrase).strip()
return text.replace(api_key, "***")[:300]
for secret in secrets.values():
if secret:
text = text.replace(str(secret), "***")
return text[:300]
async def test_openai_credential(
config: CredentialTestRequest,
) -> CredentialTestResult:
async def test_model_resource(
interface_type: str,
capability: str,
values: dict,
secrets: dict,
) -> ModelResourceTestResult:
if interface_type.startswith("xfyun-"):
return ModelResourceTestResult(
ok=True,
message="讯飞连接参数有效",
detail="鉴权字段和连接参数完整，请在语音测试页验证签名及音频链路",
)
if capability == "Realtime":
return ModelResourceTestResult(
ok=False,
message="暂不支持 Realtime 连接测试",
detail="请在助手语音测试页验证实时连接",
)
api_url = str(values.get("apiUrl") or "")
model_id = str(values.get("modelId") or "")
api_key = str(secrets.get("apiKey") or "")
headers = {"Authorization": f"Bearer {api_key}"}
started = time.perf_counter()
headers = {"Authorization": f"Bearer {config.api_key}"}
try:
async with httpx.AsyncClient(timeout=TEST_TIMEOUT_SECONDS) as client:
if config.type == "LLM":
if capability == "LLM":
response = await client.post(
_endpoint(config.api_url, "chat/completions"),
_endpoint(api_url, "chat/completions"),
headers=headers,
json={
"model": config.model_id,
"model": model_id,
"messages": [{"role": "user", "content": "Reply with OK."}],
"max_tokens": 1,
"stream": False,
},
)
elif config.type == "Embedding":
elif capability == "Embedding":
response = await client.post(
_endpoint(config.api_url, "embeddings"),
_endpoint(api_url, "embeddings"),
headers=headers,
json={"model": config.model_id, "input": "ping"},
json={"model": model_id, "input": "ping"},
)
elif config.type == "ASR":
elif capability == "ASR":
response = await client.post(
_endpoint(config.api_url, "audio/transcriptions"),
_endpoint(api_url, "audio/transcriptions"),
headers=headers,
data={
"model": config.model_id,
**({"language": config.language} if config.language else {}),
"model": model_id,
**(
{"language": str(values["language"])}
if values.get("language")
else {}
),
},
files={"file": ("test.wav", _silent_wav(), "audio/wav")},
)
elif config.type == "TTS":
elif capability == "TTS":
response = await client.post(
_endpoint(config.api_url, "audio/speech"),
_endpoint(api_url, "audio/speech"),
headers=headers,
json={
"model": config.model_id,
"model": model_id,
"input": "测试",
"voice": config.voice,
"voice": str(values.get("voice") or config.TTS_VOICE),
"response_format": "pcm",
"speed": config.speed,
"speed": float(values.get("speed") or 1),
},
)
else:
return CredentialTestResult(
return ModelResourceTestResult(
ok=False,
message="暂不支持该资源类型的连通测试",
detail=f"当前仅支持 LLM、Embedding、ASR、TTS，收到 {config.type}",
message="暂不支持该能力的连接测试",
detail=f"收到能力类型 {capability}",
)
latency_ms = round((time.perf_counter() - started) * 1000)
if response.is_success:
return CredentialTestResult(
return ModelResourceTestResult(
ok=True,
latency_ms=latency_ms,
message="连接成功",
detail=f"OpenAI 兼容接口响应正常（HTTP {response.status_code}）",
detail=f"接口响应正常（HTTP {response.status_code}）",
)
return CredentialTestResult(
return ModelResourceTestResult(
ok=False,
latency_ms=latency_ms,
message=f"连接失败（HTTP {response.status_code}）",
detail=_error_detail(response, config.api_key),
detail=_error_detail(response, secrets),
)
except httpx.TimeoutException:
return CredentialTestResult(
return ModelResourceTestResult(
ok=False,
latency_ms=round((time.perf_counter() - started) * 1000),
message="连接超时",
detail=f"服务未在 {TEST_TIMEOUT_SECONDS:g} 秒内响应",
)
except httpx.RequestError as exc:
return CredentialTestResult(
return ModelResourceTestResult(
ok=False,
latency_ms=round((time.perf_counter() - started) * 1000),
message="无法连接到模型服务",
detail=str(exc)[:300],
)
def test_xfyun_credential(config: CredentialTestRequest) -> CredentialTestResult:
"""Validate the Xfyun credential packed into the existing api_key field.
Actual signed-WebSocket synthesis/recognition is exercised by the voice
pipeline; this check deliberately avoids consuming provider quota.
"""
try:
parse_xfyun_credential(config.api_key)
except ValueError as exc:
return CredentialTestResult(
ok=False,
message="讯飞凭证格式无效",
detail=str(exc),
)
return CredentialTestResult(
ok=True,
message="讯飞凭证格式有效",
detail="请在语音测试页验证签名、识别和合成链路",
)

73
backend/services/pipecat/service_factory.py
View File

@@ -1,7 +1,7 @@
"""创建 STT / LLM / TTS 服务。
对应 dograh 的 service_factory.py,但只留一套国产栈(OpenAI 兼容),
按 provider 扩展时在这里加分支即可——这是未来接更多模型的唯一入口。
按 interface_type 扩展时在这里加分支即可——这是未来接更多模型的唯一入口。
"""
import config
@@ -14,13 +14,7 @@ from pipecat.services.openai.tts import VALID_VOICES, OpenAITTSService
from pipecat.transcriptions.language import Language
from services.pipecat.xfyun_asr import DEFAULT_XFYUN_ASR_URL, XfyunASRService
from services.pipecat.xfyun_config import (
is_super_tts,
parse_xfyun_credential,
websocket_url,
xfyun_language,
xfyun_speed,
)
from services.pipecat.xfyun_config import websocket_url, xfyun_language, xfyun_speed
from services.pipecat.xfyun_super_tts import (
DEFAULT_XFYUN_SUPER_TTS_URL,
XfyunSuperTTSService,
@@ -43,16 +37,21 @@ def create_stt(cfg: AssistantConfig):
连接信息优先用 cfg(由 config_resolver 从 DB 注入),为空回退 .env 默认。
"""
if cfg.stt_interface_type == "xfyun":
credential = parse_xfyun_credential(cfg.stt_api_key)
if cfg.stt_interface_type == "xfyun-asr":
return XfyunASRService(
app_id=credential.app_id,
api_key=credential.api_key,
api_secret=credential.api_secret,
app_id=str(cfg.stt_secrets.get("appId") or ""),
api_key=str(cfg.stt_secrets.get("apiKey") or ""),
api_secret=str(cfg.stt_secrets.get("apiSecret") or ""),
url=websocket_url(cfg.stt_base_url, DEFAULT_XFYUN_ASR_URL),
language=xfyun_language(cfg.stt_language),
sample_rate=16000,
domain=str(cfg.stt_values.get("domain") or "iat"),
accent=str(cfg.stt_values.get("accent") or "mandarin"),
frame_size=int(cfg.stt_values.get("frameSize") or 1280),
dynamic_correction=bool(cfg.stt_values.get("dynamicCorrection", False)),
)
if cfg.stt_interface_type not in {"openai-asr", "dashscope-asr"}:
raise ValueError(f"不支持的 ASR 接口类型: {cfg.stt_interface_type}")
return OpenAISTTService(
api_key=cfg.stt_api_key or config.STT_API_KEY,
@@ -66,6 +65,8 @@ def create_stt(cfg: AssistantConfig):
def create_llm(cfg: AssistantConfig):
"""DeepSeek 等,走 OpenAI 兼容的 /v1/chat/completions。"""
if cfg.llm_interface_type not in {"openai-llm", "dashscope-llm"}:
raise ValueError(f"不支持的 LLM 接口类型: {cfg.llm_interface_type}")
return OpenAILLMService(
api_key=cfg.llm_api_key or config.LLM_API_KEY,
base_url=cfg.llm_base_url or config.LLM_BASE_URL,
@@ -76,31 +77,39 @@ def create_llm(cfg: AssistantConfig):
def create_tts(cfg: AssistantConfig):
"""CosyVoice 等,走 OpenAI 兼容的 /v1/audio/speech。"""
voice = cfg.voice or config.TTS_VOICE
if cfg.tts_interface_type == "xfyun":
credential = parse_xfyun_credential(cfg.tts_api_key)
speed = xfyun_speed(cfg.tts_speed)
if is_super_tts(cfg.tts_model, cfg.tts_base_url):
return XfyunSuperTTSService(
app_id=credential.app_id,
api_key=credential.api_key,
api_secret=credential.api_secret,
voice=voice,
url=websocket_url(cfg.tts_base_url, DEFAULT_XFYUN_SUPER_TTS_URL),
sample_rate=16000,
source_sample_rate=24000,
speed=speed,
)
if cfg.tts_interface_type == "xfyun-super-tts":
return XfyunSuperTTSService(
app_id=str(cfg.tts_secrets.get("appId") or ""),
api_key=str(cfg.tts_secrets.get("apiKey") or ""),
api_secret=str(cfg.tts_secrets.get("apiSecret") or ""),
voice=voice,
url=websocket_url(cfg.tts_base_url, DEFAULT_XFYUN_SUPER_TTS_URL),
sample_rate=16000,
source_sample_rate=int(cfg.tts_values.get("sourceSampleRate") or 24000),
speed=xfyun_speed(cfg.tts_speed),
volume=int(cfg.tts_values.get("volume") or 50),
pitch=int(cfg.tts_values.get("pitch") or 50),
oral_level=str(cfg.tts_values.get("oralLevel") or "mid"),
text_aggregation_mode=str(
cfg.tts_values.get("textAggregationMode") or "token"
),
)
if cfg.tts_interface_type == "xfyun-tts":
return XfyunTTSService(
app_id=credential.app_id,
api_key=credential.api_key,
api_secret=credential.api_secret,
app_id=str(cfg.tts_secrets.get("appId") or ""),
api_key=str(cfg.tts_secrets.get("apiKey") or ""),
api_secret=str(cfg.tts_secrets.get("apiSecret") or ""),
voice=voice,
url=websocket_url(cfg.tts_base_url, DEFAULT_XFYUN_TTS_URL),
sample_rate=16000,
source_sample_rate=16000,
speed=speed,
source_sample_rate=int(cfg.tts_values.get("sourceSampleRate") or 16000),
speed=xfyun_speed(cfg.tts_speed),
volume=int(cfg.tts_values.get("volume") or 50),
pitch=int(cfg.tts_values.get("pitch") or 50),
push_stop_frames=True,
)
if cfg.tts_interface_type not in {"openai-tts", "dashscope-tts"}:
raise ValueError(f"不支持的 TTS 接口类型: {cfg.tts_interface_type}")
# Pipecat 默认只接受 OpenAI 官方音色。OpenAI 兼容服务常使用自定义 voice id,
# 注册为原样映射后仍由 OpenAI SDK 按字符串透传给供应商。

47
backend/services/pipecat/xfyun_config.py
View File

@@ -1,46 +1,7 @@
"""Parse Xfyun's three-part credential from ProviderCredential.api_key."""
"""Shared Xfyun service value normalization."""
from __future__ import annotations
import json
from dataclasses import dataclass
@dataclass(frozen=True)
class XfyunCredential:
app_id: str
api_key: str
api_secret: str
def parse_xfyun_credential(value: str) -> XfyunCredential:
"""Accept JSON in the existing api_key column.
Example:
{"appId":"...","apiKey":"...","apiSecret":"..."}
"""
try:
payload = json.loads(value)
except json.JSONDecodeError as exc:
raise ValueError(
'Xfyun API Key must be JSON: {"appId":"...","apiKey":"...","apiSecret":"..."}'
) from exc
if not isinstance(payload, dict):
raise ValueError("Xfyun API Key JSON must be an object")
credential = XfyunCredential(
app_id=str(payload.get("appId") or payload.get("app_id") or "").strip(),
api_key=str(payload.get("apiKey") or payload.get("api_key") or "").strip(),
api_secret=str(
payload.get("apiSecret") or payload.get("api_secret") or ""
).strip(),
)
if not credential.app_id or not credential.api_key or not credential.api_secret:
raise ValueError("Xfyun API Key JSON requires appId, apiKey, and apiSecret")
return credential
def websocket_url(value: str, default: str) -> str:
url = (value or default).strip()
if url.startswith("https://"):
@@ -49,12 +10,6 @@ def websocket_url(value: str, default: str) -> str:
return f"ws://{url.removeprefix('http://')}"
return url
def is_super_tts(model_id: str, api_url: str) -> bool:
model = model_id.lower().replace("-", "_")
return "super" in model or "/private/" in api_url.lower()
def xfyun_language(value: str) -> str:
normalized = (value or "zh_cn").lower().replace("-", "_")
return {"zh": "zh_cn", "en": "en_us"}.get(normalized, normalized)