diff --git a/.DS_Store b/.DS_Store index 0739929..1ba0ef3 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/backend/.env.example b/backend/.env.example index 439df45..6bd4600 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -1,28 +1,6 @@ # 复制为 .env 并填入真实值。 -# 国产栈走 OpenAI 兼容协议:每类服务给一个 base_url + api_key + 模型名即可。 - -# ---- LLM:DeepSeek(OpenAI 兼容,直连云端,只需 key) ---- -LLM_BASE_URL=https://api.deepseek.com/v1 -LLM_API_KEY=sk-your-deepseek-key -LLM_MODEL=deepseek-chat - -# ---- STT:SenseVoice / FunASR ---- -# 需要本地起一个 OpenAI 兼容的语音转写服务(/v1/audio/transcriptions), -# 例如用 funasr / sherpa-onnx / speaches 包一层。下面填那个服务地址。 -STT_BASE_URL=http://localhost:8001/v1 -STT_API_KEY=local -STT_MODEL=sensevoice - -# ---- TTS:CosyVoice ---- -# 同样需要本地起一个 OpenAI 兼容的 TTS 服务(/v1/audio/speech)。 -TTS_BASE_URL=http://localhost:8002/v1 -TTS_API_KEY=local -TTS_MODEL=cosyvoice -TTS_VOICE=中文女 - -# ---- Realtime 模式(可选,先不接也行) ---- -REALTIME_API_KEY= -REALTIME_MODEL=gpt-realtime +# 模型接入配置不再放在 .env;请在 model_resources 中维护 apiUrl/modelId/apiKey 等字段。 +# 开发环境可先执行 `make db-seed`,再到前端「组件 / 模型」里替换真实密钥。 # ---- 数据库(Postgres) ---- # 本地直连;docker compose 里则用 postgres:5432 diff --git a/backend/README.md b/backend/README.md index 3345b27..350447d 100644 --- a/backend/README.md +++ b/backend/README.md @@ -25,7 +25,7 @@ pipecat 把"管线"和"输出方式"解耦:同一条 `STT→LLM→TTS` 管线可 ``` ai-video-backend/ ├── app.py # FastAPI 入口,挂路由 + CORS -├── config.py # 读 .env,模型接口环境变量兜底 +├── settings.py # 读 .env,仅存数据库/CORS/TURN 等运行设置 ├── models.py # AssistantConfig(对齐前端 AssistantForm) ├── routes/ # 一个文件一组端点(对齐 dograh routes/) │ ├── health.py @@ -96,7 +96,7 @@ uv pip install fastapi "uvicorn[standard]" sqlalchemy asyncpg greenlet python-do # 阶段 B:做语音时再装全量(含 pipecat,需 3.10+) # uv pip install -r requirements.txt -cp .env.example .env # CRUD 阶段只需 DATABASE_URL;语音再填模型 key +cp .env.example .env # 只放数据库/CORS/TURN;模型 key 在模型资源里维护 # 起 Postgres:在 ai-video/ 下 docker compose up -d postgres cd .. make db-migrate # 首次或表结构变更后执行 Alembic 迁移 diff --git a/backend/app.py b/backend/app.py index 3683345..b5b84e1 100644 --- a/backend/app.py +++ b/backend/app.py @@ -14,7 +14,7 @@ from contextlib import asynccontextmanager -import config +import settings import uvicorn from db.session import sync_interface_definitions from fastapi import FastAPI @@ -41,7 +41,7 @@ app = FastAPI(title="AI Video Assistant 平台 - 后端", lifespan=lifespan) app.add_middleware( CORSMiddleware, - allow_origins=config.CORS_ORIGINS, + allow_origins=settings.CORS_ORIGINS, allow_credentials=True, allow_methods=["*"], allow_headers=["*"], @@ -57,4 +57,4 @@ app.include_router(voice_ws.router) if __name__ == "__main__": - uvicorn.run("app:app", host=config.HOST, port=config.PORT, reload=True) + uvicorn.run("app:app", host=settings.HOST, port=settings.PORT, reload=True) diff --git a/backend/config.py b/backend/config.py deleted file mode 100644 index 5463cee..0000000 --- a/backend/config.py +++ /dev/null @@ -1,55 +0,0 @@ -"""集中读取环境变量。所有模型接口的环境变量兜底都在这里。""" - -import os - -from dotenv import load_dotenv - -load_dotenv() - - -def _split(value: str) -> list[str]: - return [item.strip() for item in value.split(",") if item.strip()] - - -# ---- LLM(DeepSeek 等,OpenAI 兼容) ---- -LLM_BASE_URL = os.getenv("LLM_BASE_URL", "https://api.deepseek.com/v1") -LLM_API_KEY = os.getenv("LLM_API_KEY", "") -LLM_MODEL = os.getenv("LLM_MODEL", "deepseek-chat") - -# ---- STT(SenseVoice / FunASR,OpenAI 兼容) ---- -STT_BASE_URL = os.getenv("STT_BASE_URL", "http://localhost:8001/v1") -STT_API_KEY = os.getenv("STT_API_KEY", "local") -STT_MODEL = os.getenv("STT_MODEL", "sensevoice") - -# ---- TTS(CosyVoice,OpenAI 兼容) ---- -TTS_BASE_URL = os.getenv("TTS_BASE_URL", "http://localhost:8002/v1") -TTS_API_KEY = os.getenv("TTS_API_KEY", "local") -TTS_MODEL = os.getenv("TTS_MODEL", "cosyvoice") -TTS_VOICE = os.getenv("TTS_VOICE", "中文女") - -# ---- Realtime(可选) ---- -REALTIME_API_KEY = os.getenv("REALTIME_API_KEY", "") -REALTIME_MODEL = os.getenv("REALTIME_MODEL", "gpt-realtime") - -# ---- 数据库(Postgres) ---- -DATABASE_URL = os.getenv( - "DATABASE_URL", - "postgresql+asyncpg://postgres:postgres@localhost:5432/postgres", -) - -# ---- 服务 ---- -HOST = os.getenv("HOST", "0.0.0.0") -PORT = int(os.getenv("PORT", "8000")) -CORS_ORIGINS = _split( - os.getenv("CORS_ORIGINS", "http://localhost:3000,http://127.0.0.1:3000") -) - -# ---- WebRTC TURN(公网跨网语音预览;本地开发留空即可) ---- -# TURN_URLS 示例:turn:182.92.86.220:3478?transport=udp,turn:182.92.86.220:3478?transport=tcp -TURN_URLS = _split(os.getenv("TURN_URLS", "")) -# coturn --use-auth-secret 时与 --static-auth-secret 一致;后端据此签发短时凭证 -TURN_SECRET = os.getenv("TURN_SECRET", "") -# 不用 secret 时可改静态账号(需 coturn --user=...) -TURN_USERNAME = os.getenv("TURN_USERNAME", "") -TURN_PASSWORD = os.getenv("TURN_PASSWORD", "") -TURN_CREDENTIAL_TTL = int(os.getenv("TURN_CREDENTIAL_TTL", "86400")) diff --git a/backend/db/session.py b/backend/db/session.py index 936449e..8367c59 100644 --- a/backend/db/session.py +++ b/backend/db/session.py @@ -8,7 +8,7 @@ from collections.abc import AsyncGenerator import json -import config +import settings from services.interface_catalog import INTERFACE_DEFINITIONS from sqlalchemy import text from sqlalchemy.ext.asyncio import ( @@ -17,7 +17,7 @@ from sqlalchemy.ext.asyncio import ( create_async_engine, ) -engine = create_async_engine(config.DATABASE_URL, echo=False, pool_pre_ping=True) +engine = create_async_engine(settings.DATABASE_URL, echo=False, pool_pre_ping=True) SessionLocal = async_sessionmaker(engine, expire_on_commit=False) diff --git a/backend/migrations/env.py b/backend/migrations/env.py index 5bdca2a..f01528b 100644 --- a/backend/migrations/env.py +++ b/backend/migrations/env.py @@ -7,7 +7,7 @@ from sqlalchemy import pool from sqlalchemy.engine import Connection from sqlalchemy.ext.asyncio import async_engine_from_config -import config as app_config +import settings as app_settings from db.models import Base config = context.config @@ -19,7 +19,7 @@ target_metadata = Base.metadata def get_url() -> str: - return app_config.DATABASE_URL + return app_settings.DATABASE_URL def run_migrations_offline() -> None: diff --git a/backend/models.py b/backend/models.py index d393f4d..eede063 100644 --- a/backend/models.py +++ b/backend/models.py @@ -70,7 +70,7 @@ class AssistantConfig(BaseModel): fastgpt_app_id: str = "" # ---- 运行时连接信息(服务端注入,不来自浏览器) ---- - # 为空时,service_factory 会回退到 config.py 的 .env 默认值。 + # 由模型资源注入;调试 inline_config 也必须显式提供。 llm_api_key: str = "" llm_base_url: str = "" stt_api_key: str = "" diff --git a/backend/services/config_resolver.py b/backend/services/config_resolver.py index 95265bd..d9b96ea 100644 --- a/backend/services/config_resolver.py +++ b/backend/services/config_resolver.py @@ -1,10 +1,9 @@ """assistant_id → 运行时配置(把真 key 在服务端组装好)。 浏览器只传 assistant_id;真 key 在这里从 model_resources 取出注入。 -助手按 capability binding 引用资源;取不到则回退该能力默认资源,再回退 .env。 +助手按 capability binding 引用资源;取不到则回退该能力默认资源。 """ -import config from db.models import Assistant, AssistantModelBinding, ModelResource from models import AssistantConfig from sqlalchemy import select @@ -63,14 +62,14 @@ async def _agent_resource_for( ).scalar_one_or_none() -def _value(resource: ModelResource | None, key: str, default): +def _value(resource: ModelResource | None, key: str, default=""): if not resource: return default value = (resource.values or {}).get(key, default) return default if value is None else value -def _secret(resource: ModelResource | None, key: str, default: str) -> str: +def _secret(resource: ModelResource | None, key: str, default: str = "") -> str: if not resource: return default return str((resource.secrets or {}).get(key) or default) @@ -148,19 +147,15 @@ async def resolve_runtime_config( agent_interface_type=(agent_resource.interface_type if agent_resource else ""), agent_values=(agent_resource.values or {}) if agent_resource else {}, agent_secrets=(agent_resource.secrets or {}) if agent_resource else {}, - # 运行时连接信息(真 key + url):模型资源优先,否则 .env 兜底 - llm_api_key=_secret(llm_resource, "apiKey", config.LLM_API_KEY), - llm_base_url=str(_value(llm_resource, "apiUrl", config.LLM_BASE_URL)), - vision_llm_api_key=_secret( - vision_resource, "apiKey", config.LLM_API_KEY - ), - vision_llm_base_url=str( - _value(vision_resource, "apiUrl", config.LLM_BASE_URL) - ), - stt_api_key=_secret(stt_resource, "apiKey", config.STT_API_KEY), - stt_base_url=str(_value(stt_resource, "apiUrl", config.STT_BASE_URL)), - tts_api_key=_secret(tts_resource, "apiKey", config.TTS_API_KEY), - tts_base_url=str(_value(tts_resource, "apiUrl", config.TTS_BASE_URL)), + # 运行时连接信息(真 key + url):来自模型资源,不再从 .env 兜底。 + llm_api_key=_secret(llm_resource, "apiKey"), + llm_base_url=str(_value(llm_resource, "apiUrl")), + vision_llm_api_key=_secret(vision_resource, "apiKey"), + vision_llm_base_url=str(_value(vision_resource, "apiUrl")), + stt_api_key=_secret(stt_resource, "apiKey"), + stt_base_url=str(_value(stt_resource, "apiUrl")), + tts_api_key=_secret(tts_resource, "apiKey"), + tts_base_url=str(_value(tts_resource, "apiUrl")), realtime_api_key=_secret(realtime_resource, "apiKey", ""), realtime_base_url=str(_value(realtime_resource, "apiUrl", "")), ) diff --git a/backend/services/model_resource_tester.py b/backend/services/model_resource_tester.py index 0f86b44..6b96ea3 100644 --- a/backend/services/model_resource_tester.py +++ b/backend/services/model_resource_tester.py @@ -12,10 +12,10 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit import httpx from websockets.asyncio.client import connect as websocket_connect -import config from schemas import ModelResourceTestResult TEST_TIMEOUT_SECONDS = 10.0 +DEFAULT_TEST_TTS_VOICE = "alloy" def _endpoint(base_url: str, path: str) -> str: @@ -116,7 +116,7 @@ async def test_model_resource( json={ "model": model_id, "input": "测试", - "voice": str(values.get("voice") or config.TTS_VOICE), + "voice": str(values.get("voice") or DEFAULT_TEST_TTS_VOICE), "response_format": "pcm", "speed": float(values.get("speed") or 1), }, diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index 841e0d6..b2cf73f 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -11,7 +11,6 @@ import base64 from io import BytesIO from uuid import uuid4 -import config from loguru import logger from models import AssistantConfig from openai import AsyncOpenAI @@ -77,6 +76,12 @@ VISION_ANALYSIS_SYSTEM_PROMPT = ( ) +def _require(value: str, label: str) -> str: + if value: + return value + raise ValueError(f"缺少模型资源配置: {label}") + + def _vision_uses_main_llm(cfg: AssistantConfig) -> bool: """模型自己支持图片时,沿用 Pipecat 的同上下文视觉工具路径。""" return not cfg.vision_model_resource_id and cfg.llm_support_image_input @@ -107,12 +112,12 @@ async def _analyze_image_with_vision_model( extra_body = cfg.vision_llm_values.get("extraBody") extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} client = AsyncOpenAI( - api_key=cfg.vision_llm_api_key or config.LLM_API_KEY, - base_url=cfg.vision_llm_base_url or config.LLM_BASE_URL, + api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"), + base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"), ) try: response = await client.chat.completions.create( - model=cfg.vision_model or config.LLM_MODEL, + model=_require(cfg.vision_model, "Vision LLM modelId"), messages=[ {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT}, { @@ -665,9 +670,9 @@ async def run_pipeline( target = await engine.route( wf_state["current"], history, - api_key=cfg.llm_api_key or config.LLM_API_KEY, - base_url=cfg.llm_base_url or config.LLM_BASE_URL, - model=cfg.model or config.LLM_MODEL, + api_key=_require(cfg.llm_api_key, "LLM apiKey"), + base_url=_require(cfg.llm_base_url, "LLM apiUrl"), + model=_require(cfg.model, "LLM modelId"), ) if target and target != wf_state["current"]: logger.info(f"文本兜底触发转移 → {engine.name(target)}") diff --git a/backend/services/pipecat/service_factory.py b/backend/services/pipecat/service_factory.py index 89317fd..898ea3c 100644 --- a/backend/services/pipecat/service_factory.py +++ b/backend/services/pipecat/service_factory.py @@ -4,7 +4,6 @@ 按 interface_type 扩展时在这里加分支即可——这是未来接更多模型的唯一入口。 """ -import config from loguru import logger from models import AssistantConfig @@ -27,6 +26,12 @@ from services.pipecat.xfyun_tts import DEFAULT_XFYUN_TTS_URL, XfyunTTSService TTS_STOP_FRAME_TIMEOUT_S = 1.0 +def _require(value: str, label: str) -> str: + if value: + return value + raise ValueError(f"缺少模型资源配置: {label}") + + def _language(value: str) -> Language | None: if not value: return None @@ -40,7 +45,7 @@ def _language(value: str) -> Language | None: def create_stt(cfg: AssistantConfig): """SenseVoice / FunASR 等,走 OpenAI 兼容的 /v1/audio/transcriptions。 - 连接信息优先用 cfg(由 config_resolver 从 DB 注入),为空回退 .env 默认。 + 连接信息来自 cfg(由 config_resolver 从 DB 注入,或调试时 inline_config 传入)。 """ if cfg.stt_interface_type == "xfyun-asr": return XfyunASRService( @@ -59,10 +64,10 @@ def create_stt(cfg: AssistantConfig): raise ValueError(f"不支持的 ASR 接口类型: {cfg.stt_interface_type}") return OpenAISTTService( - api_key=cfg.stt_api_key or config.STT_API_KEY, - base_url=cfg.stt_base_url or config.STT_BASE_URL, + api_key=_require(cfg.stt_api_key, "ASR apiKey"), + base_url=_require(cfg.stt_base_url, "ASR apiUrl"), settings=OpenAISTTService.Settings( - model=cfg.asr or config.STT_MODEL, + model=_require(cfg.asr, "ASR modelId"), language=_language(cfg.stt_language), ), ) @@ -75,10 +80,10 @@ def create_llm(cfg: AssistantConfig): extra_body = cfg.llm_values.get("extraBody") extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} return OpenAILLMService( - api_key=cfg.llm_api_key or config.LLM_API_KEY, - base_url=cfg.llm_base_url or config.LLM_BASE_URL, + api_key=_require(cfg.llm_api_key, "LLM apiKey"), + base_url=_require(cfg.llm_base_url, "LLM apiUrl"), settings=OpenAILLMService.Settings( - model=cfg.model or config.LLM_MODEL, + model=_require(cfg.model, "LLM modelId"), extra=extra, ), ) @@ -86,7 +91,7 @@ def create_llm(cfg: AssistantConfig): def create_tts(cfg: AssistantConfig): """CosyVoice 等,走 OpenAI 兼容的 /v1/audio/speech。""" - voice = cfg.voice or config.TTS_VOICE + voice = _require(cfg.voice, "TTS voice") if cfg.tts_interface_type == "xfyun-super-tts": return XfyunSuperTTSService( app_id=str(cfg.tts_secrets.get("appId") or ""), @@ -126,11 +131,11 @@ def create_tts(cfg: AssistantConfig): # 注册为原样映射后仍由 OpenAI SDK 按字符串透传给供应商。 VALID_VOICES.setdefault(voice, voice) return OpenAITTSService( - api_key=cfg.tts_api_key or config.TTS_API_KEY, - base_url=cfg.tts_base_url or config.TTS_BASE_URL, + api_key=_require(cfg.tts_api_key, "TTS apiKey"), + base_url=_require(cfg.tts_base_url, "TTS apiUrl"), stop_frame_timeout_s=TTS_STOP_FRAME_TIMEOUT_S, settings=OpenAITTSService.Settings( - model=cfg.tts_model or config.TTS_MODEL, + model=_require(cfg.tts_model, "TTS modelId"), voice=voice, speed=cfg.tts_speed, ), @@ -143,9 +148,9 @@ def create_realtime_service(cfg: AssistantConfig): from services.pipecat.stepfun_realtime import StepFunRealtimeService return StepFunRealtimeService( - api_key=cfg.realtime_api_key, - model=cfg.realtimeModel, - base_url=cfg.realtime_base_url, + api_key=_require(cfg.realtime_api_key, "Realtime apiKey"), + model=_require(cfg.realtimeModel, "Realtime modelId"), + base_url=_require(cfg.realtime_base_url, "Realtime apiUrl"), instructions=cfg.prompt, voice=str(cfg.realtime_values.get("voice") or "linjiajiejie"), input_sample_rate=int( diff --git a/backend/services/webrtc_ice.py b/backend/services/webrtc_ice.py index 5b64ed6..307ffbe 100644 --- a/backend/services/webrtc_ice.py +++ b/backend/services/webrtc_ice.py @@ -7,28 +7,28 @@ import hashlib import hmac import time -import config +import settings STUN_URL = "stun:stun.l.google.com:19302" def _turn_credentials() -> tuple[str, str] | None: """Return (username, credential) for TURN, or None when TURN is not configured.""" - if not config.TURN_URLS: + if not settings.TURN_URLS: return None - if config.TURN_SECRET: - expiry = int(time.time()) + config.TURN_CREDENTIAL_TTL + if settings.TURN_SECRET: + expiry = int(time.time()) + settings.TURN_CREDENTIAL_TTL username = f"{expiry}:ai-video" credential = base64.b64encode( hmac.new( - config.TURN_SECRET.encode("utf-8"), + settings.TURN_SECRET.encode("utf-8"), username.encode("utf-8"), hashlib.sha1, ).digest() ).decode("utf-8") return username, credential - if config.TURN_USERNAME and config.TURN_PASSWORD: - return config.TURN_USERNAME, config.TURN_PASSWORD + if settings.TURN_USERNAME and settings.TURN_PASSWORD: + return settings.TURN_USERNAME, settings.TURN_PASSWORD return None @@ -39,7 +39,7 @@ def client_ice_servers() -> list[dict]: if not creds: return servers username, credential = creds - for url in config.TURN_URLS: + for url in settings.TURN_URLS: servers.append({"urls": url, "username": username, "credential": credential}) return servers @@ -53,7 +53,7 @@ def aiortc_ice_servers() -> list: if not creds: return servers username, credential = creds - for url in config.TURN_URLS: + for url in settings.TURN_URLS: servers.append( RTCIceServer(urls=url, username=username, credential=credential) ) diff --git a/backend/settings.py b/backend/settings.py new file mode 100644 index 0000000..6e79a08 --- /dev/null +++ b/backend/settings.py @@ -0,0 +1,39 @@ +"""Runtime settings for backend infrastructure. + +Model provider credentials live in ``model_resources`` and are resolved per +assistant. Keep this module limited to process-level settings such as database, +CORS, server bind address, and WebRTC ICE/TURN configuration. +""" + +import os + +from dotenv import load_dotenv + +load_dotenv() + + +def _split(value: str) -> list[str]: + return [item.strip() for item in value.split(",") if item.strip()] + + +# ---- Database ---- +DATABASE_URL = os.getenv( + "DATABASE_URL", + "postgresql+asyncpg://postgres:postgres@localhost:5432/postgres", +) + +# ---- Service ---- +HOST = os.getenv("HOST", "0.0.0.0") +PORT = int(os.getenv("PORT", "8000")) +CORS_ORIGINS = _split( + os.getenv("CORS_ORIGINS", "http://localhost:3000,http://127.0.0.1:3000") +) + +# ---- WebRTC TURN ---- +# TURN_URLS example: +# turn:182.92.86.220:3478?transport=udp,turn:182.92.86.220:3478?transport=tcp +TURN_URLS = _split(os.getenv("TURN_URLS", "")) +TURN_SECRET = os.getenv("TURN_SECRET", "") +TURN_USERNAME = os.getenv("TURN_USERNAME", "") +TURN_PASSWORD = os.getenv("TURN_PASSWORD", "") +TURN_CREDENTIAL_TTL = int(os.getenv("TURN_CREDENTIAL_TTL", "86400")) diff --git a/scripts/audio-sender-sine-tester.js b/scripts/tampermonkey/audio-sender-sine-tester.js similarity index 100% rename from scripts/audio-sender-sine-tester.js rename to scripts/tampermonkey/audio-sender-sine-tester.js diff --git a/scripts/audio-track-probe.js b/scripts/tampermonkey/audio-track-probe.js similarity index 100% rename from scripts/audio-track-probe.js rename to scripts/tampermonkey/audio-track-probe.js diff --git a/scripts/copilot-element2-two-party.js b/scripts/tampermonkey/copilot-element2-two-party.js similarity index 100% rename from scripts/copilot-element2-two-party.js rename to scripts/tampermonkey/copilot-element2-two-party.js diff --git a/scripts/copilot.js b/scripts/tampermonkey/copilot.js similarity index 100% rename from scripts/copilot.js rename to scripts/tampermonkey/copilot.js diff --git a/scripts/video-track-probe.js b/scripts/tampermonkey/video-track-probe.js similarity index 100% rename from scripts/video-track-probe.js rename to scripts/tampermonkey/video-track-probe.js