Initial commit: AI Video Assistant fullstack platform.

Add pipecat-based backend with WebRTC/WS voice routes, Next.js frontend, and Docker Compose orchestration.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/routes/assistants.py

@@ -0,0 +1,87 @@
+"""助手 CRUD。前端「助手列表 / 创建 / 编辑」对接这里。
+
+助手配置不含 key,所以无需打码。
+"""
+
+import uuid
+
+from db.models import Assistant
+from db.session import get_session
+from fastapi import APIRouter, Depends, HTTPException
+from schemas import AssistantOut, AssistantUpsert
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+router = APIRouter(prefix="/api/assistants", tags=["assistants"])
+
+
+def _to_out(a: Assistant) -> AssistantOut:
+    return AssistantOut(
+        id=a.id,
+        name=a.name,
+        greeting=a.greeting,
+        prompt=a.prompt,
+        runtime_mode=a.runtime_mode,  # type: ignore[arg-type]
+        model=a.model,
+        asr=a.asr,
+        voice=a.voice,
+        enable_interrupt=a.enable_interrupt,
+        updated_at=a.updated_at.isoformat() if a.updated_at else None,
+    )
+
+
+@router.get("", response_model=list[AssistantOut])
+async def list_assistants(session: AsyncSession = Depends(get_session)):
+    rows = (
+        await session.execute(select(Assistant).order_by(Assistant.updated_at.desc()))
+    ).scalars().all()
+    return [_to_out(a) for a in rows]
+
+
+@router.post("", response_model=AssistantOut)
+async def create_assistant(
+    body: AssistantUpsert, session: AsyncSession = Depends(get_session)
+):
+    a = Assistant(id=f"asst_{uuid.uuid4().hex[:12]}", **body.model_dump())
+    session.add(a)
+    await session.commit()
+    await session.refresh(a)
+    return _to_out(a)
+
+
+@router.get("/{assistant_id}", response_model=AssistantOut)
+async def get_assistant(
+    assistant_id: str, session: AsyncSession = Depends(get_session)
+):
+    a = await session.get(Assistant, assistant_id)
+    if not a:
+        raise HTTPException(404, "助手不存在")
+    return _to_out(a)
+
+
+@router.put("/{assistant_id}", response_model=AssistantOut)
+async def update_assistant(
+    assistant_id: str,
+    body: AssistantUpsert,
+    session: AsyncSession = Depends(get_session),
+):
+    a = await session.get(Assistant, assistant_id)
+    if not a:
+        raise HTTPException(404, "助手不存在")
+    for k, v in body.model_dump().items():
+        setattr(a, k, v)
+    await session.commit()
+    await session.refresh(a)
+    return _to_out(a)
+
+
+@router.delete("/{assistant_id}")
+async def delete_assistant(
+    assistant_id: str, session: AsyncSession = Depends(get_session)
+):
+    a = await session.get(Assistant, assistant_id)
+    if not a:
+        raise HTTPException(404, "助手不存在")
+    await session.delete(a)
+    await session.commit()
+    return {"ok": True}

backend/routes/credentials.py

@@ -0,0 +1,106 @@
+"""模型资源凭证 CRUD。前端 ComponentsModelsPage 对接这里。
+
+字段对齐前端 ModelResource。读:api_key 打码;写:占位符表示不改(写时哨兵)。
+设默认时清掉同 type 其它默认。
+"""
+
+import uuid
+
+from db.models import ProviderCredential
+from db.session import get_session
+from fastapi import APIRouter, Depends, HTTPException
+from schemas import CredentialOut, CredentialUpsert
+from services.masking import mask, resolve_incoming_key
+from sqlalchemy import select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+router = APIRouter(prefix="/api/credentials", tags=["credentials"])
+
+
+def _to_out(c: ProviderCredential) -> CredentialOut:
+    return CredentialOut(
+        id=c.id,
+        name=c.name,
+        model_id=c.model_id,
+        type=c.type,
+        interface_type=c.interface_type,
+        api_url=c.api_url,
+        api_key=mask(c.api_key),  # 永远打码
+        is_default=c.is_default,
+    )
+
+
+async def _clear_other_defaults(session: AsyncSession, type_: str, keep_id: str):
+    await session.execute(
+        update(ProviderCredential)
+        .where(ProviderCredential.type == type_, ProviderCredential.id != keep_id)
+        .values(is_default=False)
+    )
+
+
+@router.get("", response_model=list[CredentialOut])
+async def list_credentials(session: AsyncSession = Depends(get_session)):
+    rows = (
+        await session.execute(
+            select(ProviderCredential).order_by(ProviderCredential.type)
+        )
+    ).scalars().all()
+    return [_to_out(c) for c in rows]
+
+
+@router.post("", response_model=CredentialOut)
+async def create_credential(
+    body: CredentialUpsert, session: AsyncSession = Depends(get_session)
+):
+    c = ProviderCredential(
+        id=f"model_{uuid.uuid4().hex[:12]}",
+        name=body.name,
+        model_id=body.model_id,
+        type=body.type,
+        interface_type=body.interface_type,
+        api_url=body.api_url,
+        api_key=resolve_incoming_key(body.api_key, ""),
+        is_default=body.is_default,
+    )
+    session.add(c)
+    if c.is_default:
+        await _clear_other_defaults(session, c.type, c.id)
+    await session.commit()
+    await session.refresh(c)
+    return _to_out(c)
+
+
+@router.put("/{cred_id}", response_model=CredentialOut)
+async def update_credential(
+    cred_id: str,
+    body: CredentialUpsert,
+    session: AsyncSession = Depends(get_session),
+):
+    c = await session.get(ProviderCredential, cred_id)
+    if not c:
+        raise HTTPException(404, "凭证不存在")
+    c.name = body.name
+    c.model_id = body.model_id
+    c.type = body.type
+    c.interface_type = body.interface_type
+    c.api_url = body.api_url
+    c.is_default = body.is_default
+    # 写时哨兵:打码占位符 → 保留旧 key
+    c.api_key = resolve_incoming_key(body.api_key, c.api_key)
+    if c.is_default:
+        await _clear_other_defaults(session, c.type, c.id)
+    await session.commit()
+    await session.refresh(c)
+    return _to_out(c)
+
+
+@router.delete("/{cred_id}")
+async def delete_credential(
+    cred_id: str, session: AsyncSession = Depends(get_session)
+):
+    c = await session.get(ProviderCredential, cred_id)
+    if not c:
+        raise HTTPException(404, "凭证不存在")
+    await session.delete(c)
+    await session.commit()
+    return {"ok": True}

backend/routes/health.py

@@ -0,0 +1,8 @@
+from fastapi import APIRouter
+
+router = APIRouter()
+
+
+@router.get("/health")
+async def health():
+    return {"status": "ok"}

backend/routes/voice_webrtc.py

@@ -0,0 +1,117 @@
+"""WebRTC 输出:SmallWebRTC 信令握手。
+
+参考 dograh 的 webrtc_signaling.py,砍掉鉴权/配额/DB/org/ICE 过滤策略/TURN。
+握手消息:
+    client → {type:"offer",         payload:{pc_id, sdp, type, config}}
+    server → {type:"answer",        payload:{pc_id, sdp, type}}
+    both   → {type:"ice-candidate", payload:{pc_id, candidate:{...}}}
+"""
+
+import asyncio
+
+from db.session import SessionLocal
+from fastapi import APIRouter, WebSocket
+from loguru import logger
+from models import AssistantConfig, SignalingOffer
+from services.config_resolver import resolve_runtime_config
+from starlette.websockets import WebSocketDisconnect, WebSocketState
+
+# 注意:pipecat / aiortc 都是重依赖(语音才用),改成函数内"惰性导入",
+# 这样不装 pipecat 也能启动后端、验证 CRUD。语音真正用到时才加载。
+
+router = APIRouter()
+
+
+def _ice_servers():
+    from aiortc import RTCIceServer
+
+    # 本地只用 STUN;公网部署在此追加 TURN(参考 dograh get_ice_servers)
+    return [RTCIceServer(urls="stun:stun.l.google.com:19302")]
+
+
+@router.websocket("/ws/voice")
+async def voice_signaling(websocket: WebSocket):
+    await websocket.accept()
+    peers: dict = {}
+    try:
+        while True:
+            message = await websocket.receive_json()
+            if message.get("type") == "offer":
+                await _handle_offer(websocket, message.get("payload", {}), peers)
+            elif message.get("type") == "ice-candidate":
+                await _handle_ice(message.get("payload", {}), peers)
+    except WebSocketDisconnect:
+        logger.info("WebRTC 信令断开")
+    except Exception as e:
+        logger.error(f"WebRTC 信令出错: {e}")
+    finally:
+        for pc in peers.values():
+            await pc.disconnect()
+
+
+async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
+    """优先用 assistant_id 从 DB 解析(含真 key);否则用调试内联配置。"""
+    if offer.assistant_id:
+        async with SessionLocal() as session:
+            return await resolve_runtime_config(session, offer.assistant_id)
+    if offer.inline_config:
+        return offer.inline_config
+    raise ValueError("offer 缺少 assistant_id 或 inline_config")
+
+
+async def _handle_offer(websocket, payload, peers):
+    from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
+    from services.pipecat.pipeline import run_pipeline
+    from services.pipecat.transports import build_webrtc_transport
+
+    offer = SignalingOffer(**payload)
+    pc_id = offer.pc_id
+
+    if pc_id and pc_id in peers:
+        pc = peers[pc_id]
+        await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False)
+    else:
+        cfg = await _resolve_config(offer)  # 解析放在建连前,配置错就别建连
+        pc = SmallWebRTCConnection(ice_servers=_ice_servers())
+        if pc_id:
+            pc._pc_id = pc_id
+        await pc.initialize(sdp=offer.sdp, type=offer.type)
+        peers[pc.pc_id] = pc
+
+        @pc.event_handler("closed")
+        async def _on_closed(conn: SmallWebRTCConnection):
+            peers.pop(conn.pc_id, None)
+
+        # 后台跑管线:WebRTC transport + 解析出的运行时配置
+        transport = build_webrtc_transport(pc)
+        asyncio.create_task(run_pipeline(transport, cfg))
+
+    answer = pc.get_answer()
+    if websocket.application_state == WebSocketState.CONNECTED:
+        await websocket.send_json(
+            {
+                "type": "answer",
+                "payload": {
+                    "pc_id": answer["pc_id"],
+                    "sdp": answer["sdp"],
+                    "type": answer["type"],
+                },
+            }
+        )
+
+
+async def _handle_ice(payload, peers):
+    from aiortc.sdp import candidate_from_sdp
+
+    pc_id = payload.get("pc_id")
+    candidate_data = payload.get("candidate")
+    pc = peers.get(pc_id) if pc_id else None
+    if not pc or not candidate_data:
+        return
+    try:
+        candidate = candidate_from_sdp(candidate_data.get("candidate", ""))
+        candidate.sdpMid = candidate_data.get("sdpMid")
+        candidate.sdpMLineIndex = candidate_data.get("sdpMLineIndex")
+        await pc.add_ice_candidate(candidate)
+    except Exception as e:
+        logger.error(f"添加 ICE candidate 失败: {e}")

backend/routes/voice_ws.py

@@ -0,0 +1,50 @@
+"""WS 输出:裸 WebSocket 音频流(第二种输出方式)。
+
+比 WebRTC 简单——没有 SDP/ICE/STUN/TURN,一条 WS 直接收发音频帧。
+适合:服务端对接、话务网关、自定义客户端、调试。
+
+约定:连接建立后,**第一条文本消息**发 JSON 启动参数:
+    {"assistant_id": "asst_xxx"}            # 推荐:key 服务端解析
+    {"inline_config": {...AssistantConfig}} # 调试:内联
+之后的二进制消息即音频帧(protobuf,与 transports.py serializer 对应)。
+"""
+
+import json
+
+from db.session import SessionLocal
+from fastapi import APIRouter, WebSocket
+from loguru import logger
+from models import AssistantConfig
+from services.config_resolver import resolve_runtime_config
+from starlette.websockets import WebSocketDisconnect
+
+# pipecat 重依赖,惰性导入(见 voice_webrtc.py 说明)
+
+router = APIRouter()
+
+
+async def _resolve_start_config(raw: str) -> AssistantConfig:
+    data = json.loads(raw)
+    if data.get("assistant_id"):
+        async with SessionLocal() as session:
+            return await resolve_runtime_config(session, data["assistant_id"])
+    if data.get("inline_config"):
+        return AssistantConfig(**data["inline_config"])
+    raise ValueError("启动参数缺少 assistant_id 或 inline_config")
+
+
+@router.websocket("/ws/stream")
+async def voice_stream(websocket: WebSocket):
+    from services.pipecat.pipeline import run_pipeline
+    from services.pipecat.transports import build_ws_transport
+
+    await websocket.accept()
+    try:
+        cfg = await _resolve_start_config(await websocket.receive_text())
+        transport = build_ws_transport(websocket)
+        # 直接 await:管线持续读这条 WS 的音频帧,直到对端断开
+        await run_pipeline(transport, cfg)
+    except WebSocketDisconnect:
+        logger.info("WS 音频流断开")
+    except Exception as e:
+        logger.error(f"WS 音频流出错: {e}")