From c5db918830ae616782a510a50a0ea7868270fd00 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Tue, 7 Jul 2026 19:41:45 +0800 Subject: [PATCH] Add support for image input in model resources and enhance related configurations - Introduce a new `support_image_input` field in model resources, allowing models to indicate support for image input. - Update the backend models, schemas, and database seed scripts to accommodate the new field. - Enhance the AssistantConfig and related routes to handle image input capabilities, ensuring proper validation and error handling. - Modify the frontend components to include toggles for enabling visual understanding and filtering models based on image input support. - Implement necessary adjustments in the voice preview and pipeline to integrate video stream handling alongside audio functionalities. --- .DS_Store | Bin 0 -> 8196 bytes .env | 4 + backend/db/models.py | 1 + backend/db/seed_model_resources.sql | 29 ++-- backend/models.py | 2 + backend/routes/model_registry.py | 8 + backend/routes/voice_webrtc.py | 11 +- backend/schemas.py | 1 + backend/services/config_resolver.py | 3 + backend/services/interface_catalog.py | 8 - backend/services/pipecat/pipeline.py | 114 ++++++++++-- backend/services/pipecat/service_factory.py | 7 +- backend/services/pipecat/transports.py | 11 +- .../src/components/pages/AssistantPage.tsx | 164 +++++++++++------- .../components/pages/ComponentsModelsPage.tsx | 120 ++++++++++++- frontend/src/hooks/use-camera-preview.ts | 18 +- frontend/src/hooks/use-voice-preview.ts | 25 ++- frontend/src/lib/api.ts | 1 + 18 files changed, 411 insertions(+), 116 deletions(-) create mode 100644 .DS_Store create mode 100644 .env diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..07399299453031bf41cfe8c5dde4a75d71b9e87c GIT binary patch literal 8196 zcmeHMy>HV%6n~c%5>ucqZJ~gWEV3X`N?TYega`oy6@pfzDlHU(?OcLY=Pt6{mQq#9 z4h!rcR#rB|!pOqPhWHa;WrBAfp@{&LP9TE7iILw9^?f50p_3tV^D|6bl7EW1`GlQ0fT@+z#w1{_!|(wXSSf+FP{5K zYkGr#LEyhcKxnC83Px0O{jsUeLU$xOBys>hwUgE73;` zqTEsM2y^5G&6|oVcOc3gh@P3~6AF>Dqn{(qffN)sy+Obr(2amNyIXJ`F2g0b-hO`P zz87*2wW6Sk*v7s|LJ1%X9=PDcGEiXRwBKRM=rLe4=RkjaqY)p(#PK-LIrY1H=osFr<@logW)a2%-l^f2VJaePG z9M&0sgh?O~y(5BH%658;7hQ=3{;E!lyl*A;$+mTBIcrzn?mc_^?SXxR149GD!^8Uz zj2t|4_)ys%^jv?r7KvNFz(XDt!xehF76p}3!}SAJTS(}Gq!c`)WqV)B^aicm-5Q!( zZ=lW4)_@FmIii%wcr(>%Op}E+d9iHw$9}}aaF)kG5^%P-6eHl%39V1FD#saD5$u_U zP{s2!ijjBuEG)WF$diQE7XqI%b?5yuEMCq=RD{f*`=Sud7p}I1 zV@j|guLJ~ns3mDm@^9%2DbF#^sFg3wDvgB;avZDe8q8zuJcK9k3|_(;cn2ThGkhgk zVw0ofI5|zul5^w=nI^Z0Pwta|L@Ge0wb)10b*m>aN?nV+T8dc6Sq*i*b|ncxO^~o_DlT2H6DDF@ p9>(jWg2@SAVYsfU7j{+s8&{N4WR-FIpAbd>-A literal 0 HcmV?d00001 diff --git a/.env b/.env new file mode 100644 index 0000000..b8dc1de --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +# docker compose 变量(复制为项目根 .env)。公网 WebRTC 需配合 --profile remote。 +PUBLIC_IP=182.92.86.220 +TURN_SECRET=change-me-to-a-long-random-string +TURN_URLS=turn:182.92.86.220:3478?transport=udp,turn:182.92.86.220:3478?transport=tcp diff --git a/backend/db/models.py b/backend/db/models.py index b983fe9..cea76a2 100644 --- a/backend/db/models.py +++ b/backend/db/models.py @@ -50,6 +50,7 @@ class ModelResource(Base): ) values: Mapped[dict] = mapped_column(JSONB, default=dict) secrets: Mapped[dict] = mapped_column(JSONB, default=dict) + support_image_input: Mapped[bool] = mapped_column(Boolean, default=False) enabled: Mapped[bool] = mapped_column(Boolean, default=True) is_default: Mapped[bool] = mapped_column(Boolean, default=False) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) diff --git a/backend/db/seed_model_resources.sql b/backend/db/seed_model_resources.sql index 5181741..e5c45ec 100644 --- a/backend/db/seed_model_resources.sql +++ b/backend/db/seed_model_resources.sql @@ -1,43 +1,46 @@ -- 模型资源种子数据。依赖应用启动时写入 interface_definitions。 INSERT INTO model_resources - (id, name, capability, interface_type, values, secrets, enabled, is_default) + (id, name, capability, interface_type, values, secrets, support_image_input, enabled, is_default) VALUES ('model_001', 'DeepSeek-Chat', 'LLM', 'openai-llm', '{"modelId":"deepseek-chat","apiUrl":"https://api.deepseek.com/v1","temperature":0.7}', - '{"apiKey":"replace-me"}', TRUE, TRUE), + '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE), ('model_002', 'SiliconFlow-TeleSpeechASR', 'ASR', 'openai-asr', '{"modelId":"TeleAI/TeleSpeechASR","apiUrl":"https://api.siliconflow.cn/v1","language":"zh"}', - '{"apiKey":"replace-me"}', TRUE, FALSE), + '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE), ('model_003', 'SiliconFlow-Qwen3-Embedding-4B', 'Embedding', 'openai-embedding', '{"modelId":"Qwen/Qwen3-Embedding-4B","apiUrl":"https://api.siliconflow.cn/v1"}', - '{"apiKey":"replace-me"}', TRUE, TRUE), + '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE), ('model_004', 'SiliconFlow-CosyVoice2-0.5B', 'TTS', 'openai-tts', '{"modelId":"FunAudioLLM/CosyVoice2-0.5B","apiUrl":"https://api.siliconflow.cn/v1","voice":"FunAudioLLM/CosyVoice2-0.5B:anna","speed":1.0,"sourceSampleRate":24000}', - '{"apiKey":"replace-me"}', TRUE, FALSE), + '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE), ('model_005', '讯飞语音识别', 'ASR', 'xfyun-asr', '{"apiUrl":"https://iat-api.xfyun.cn/v2/iat","language":"zh_cn","domain":"iat","accent":"mandarin","dynamicCorrection":false,"frameSize":1280}', - '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE), + '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', FALSE, TRUE, TRUE), ('model_006', 'Paraformer 识别', 'ASR', 'dashscope-asr', '{"modelId":"paraformer-realtime-v2","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/asr","language":"zh"}', - '{"apiKey":"replace-me"}', TRUE, FALSE), + '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE), ('model_007', '讯飞语音合成', 'TTS', 'xfyun-tts', '{"apiUrl":"https://tts-api.xfyun.cn/v2/tts","voice":"xiaoyan","speed":50,"volume":50,"pitch":50,"sourceSampleRate":16000}', - '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', TRUE, TRUE), + '{"appId":"replace-me","apiKey":"replace-me","apiSecret":"replace-me"}', FALSE, TRUE, TRUE), ('model_008', 'CosyVoice 合成', 'TTS', 'dashscope-tts', '{"modelId":"cosyvoice-v1","apiUrl":"https://dashscope.aliyuncs.com/api/v1/services/audio/tts","voice":"longxiaochun"}', - '{"apiKey":"replace-me"}', TRUE, FALSE), + '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE), ('model_009', 'GPT Realtime', 'Realtime', 'openai-realtime', '{"modelId":"gpt-4o-realtime-preview","apiUrl":"https://api.openai.com/v1/realtime"}', - '{"apiKey":"replace-me"}', TRUE, TRUE), + '{"apiKey":"replace-me"}', FALSE, TRUE, TRUE), ('model_010', 'Gemini Live', 'Realtime', 'gemini-realtime', '{"modelId":"gemini-2.0-flash-live","apiUrl":"https://generativelanguage.googleapis.com/v1beta"}', - '{"apiKey":"replace-me"}', TRUE, FALSE), + '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE), ('model_011', 'text-embedding-3', 'Embedding', 'openai-embedding', '{"modelId":"text-embedding-3-small","apiUrl":"https://api.openai.com/v1/embeddings"}', - '{"apiKey":"replace-me"}', TRUE, FALSE), + '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE), ('model_012', 'StepAudio 2.5 Realtime', 'Realtime', 'stepfun-realtime', '{"modelId":"stepaudio-2.5-realtime","apiUrl":"wss://api.stepfun.com/v1/realtime","voice":"linjiajiejie","inputSampleRate":24000,"outputSampleRate":24000,"prefixPaddingMs":500,"silenceDurationMs":300,"energyAwakenessThreshold":2500}', - '{"apiKey":"replace-me"}', TRUE, FALSE) + '{"apiKey":"replace-me"}', FALSE, TRUE, FALSE), + ('model_013', 'doubao-seed-2-0-mini-260428', 'LLM', 'openai-llm', + '{"modelId":"doubao-seed-2-0-mini-260428","apiUrl":"https://ark.cn-beijing.volces.com/api/v3","temperature":0.7,"extraBody":{"thinking":{"type":"disabled"}}}', + '{"apiKey":"replace-me"}', TRUE, TRUE, FALSE) -- Seed defaults must never overwrite resources configured through the UI. ON CONFLICT (id) DO NOTHING; diff --git a/backend/models.py b/backend/models.py index ba8aca4..f648e8f 100644 --- a/backend/models.py +++ b/backend/models.py @@ -41,6 +41,7 @@ class AssistantConfig(BaseModel): tts_interface_type: str = "openai-tts" llm_values: dict = {} llm_secrets: dict = {} + llm_support_image_input: bool = False stt_values: dict = {} stt_secrets: dict = {} tts_values: dict = {} @@ -80,3 +81,4 @@ class SignalingOffer(BaseModel): type: str assistant_id: str | None = None inline_config: AssistantConfig | None = None + vision_enabled: bool = False diff --git a/backend/routes/model_registry.py b/backend/routes/model_registry.py index 2dcb2f7..e7867b6 100644 --- a/backend/routes/model_registry.py +++ b/backend/routes/model_registry.py @@ -53,6 +53,7 @@ def _resource_out(row: ModelResource) -> ModelResourceOut: interface_type=row.interface_type, values=row.values or {}, secrets=mask_secrets(row.secrets or {}), + support_image_input=row.support_image_input, enabled=row.enabled, is_default=row.is_default, updated_at=row.updated_at.isoformat() if row.updated_at else None, @@ -140,6 +141,9 @@ async def create_model_resource( interface_type=definition.interface_type, values=body.values, secrets=secrets, + support_image_input=body.support_image_input + if definition.capability == "LLM" + else False, enabled=body.enabled, is_default=body.is_default, ) @@ -200,6 +204,7 @@ async def duplicate_model_resource( interface_type=source.interface_type, values=dict(source.values or {}), secrets=dict(source.secrets or {}), + support_image_input=source.support_image_input, enabled=source.enabled, is_default=False, ) @@ -223,6 +228,9 @@ async def update_model_resource( row.interface_type = definition.interface_type row.values = body.values row.secrets = secrets + row.support_image_input = ( + body.support_image_input if definition.capability == "LLM" else False + ) row.enabled = body.enabled row.is_default = body.is_default if row.is_default: diff --git a/backend/routes/voice_webrtc.py b/backend/routes/voice_webrtc.py index d0e131c..f947f75 100644 --- a/backend/routes/voice_webrtc.py +++ b/backend/routes/voice_webrtc.py @@ -87,6 +87,8 @@ async def _handle_offer(websocket, payload, peers): await pc.renegotiate(sdp=offer.sdp, type=offer.type, restart_pc=False) else: cfg = await _resolve_config(offer) # 解析放在建连前,配置错就别建连 + if offer.vision_enabled and not cfg.llm_support_image_input: + raise ValueError("当前大语言模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM") pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers()) if pc_id: pc._pc_id = pc_id @@ -98,8 +100,13 @@ async def _handle_offer(websocket, payload, peers): peers.pop(conn.pc_id, None) # 后台跑管线:WebRTC transport + 解析出的运行时配置 - transport = build_webrtc_transport(pc) - asyncio.create_task(run_pipeline(transport, cfg)) + transport = build_webrtc_transport( + pc, + video_in_enabled=offer.vision_enabled, + ) + asyncio.create_task( + run_pipeline(transport, cfg, vision_enabled=offer.vision_enabled) + ) answer = pc.get_answer() if websocket.application_state == WebSocketState.CONNECTED: diff --git a/backend/schemas.py b/backend/schemas.py index 51a055d..ff29562 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -111,6 +111,7 @@ class ModelResourceUpsert(CamelModel): interface_type: str values: dict[str, Any] = Field(default_factory=dict) secrets: dict[str, Any] = Field(default_factory=dict) + support_image_input: bool = False enabled: bool = True is_default: bool = False diff --git a/backend/services/config_resolver.py b/backend/services/config_resolver.py index 6599086..1ab155a 100644 --- a/backend/services/config_resolver.py +++ b/backend/services/config_resolver.py @@ -86,6 +86,9 @@ async def resolve_runtime_config( realtimeModel=str(_value(realtime_resource, "modelId", "")), llm_values=(llm_resource.values or {}) if llm_resource else {}, llm_secrets=(llm_resource.secrets or {}) if llm_resource else {}, + llm_support_image_input=( + bool(llm_resource.support_image_input) if llm_resource else False + ), stt_values=(stt_resource.values or {}) if stt_resource else {}, stt_secrets=(stt_resource.secrets or {}) if stt_resource else {}, tts_values=(tts_resource.values or {}) if tts_resource else {}, diff --git a/backend/services/interface_catalog.py b/backend/services/interface_catalog.py index 9e1b8cb..a97362b 100644 --- a/backend/services/interface_catalog.py +++ b/backend/services/interface_catalog.py @@ -138,13 +138,6 @@ INTERFACE_DEFINITIONS: list[dict] = [ field("textAggregationMode", "Text Aggregation Mode", default="token"), ], }, - { - "interface_type": "dashscope-llm", - "name": "DashScope LLM", - "capability": "LLM", - "fields": OPENAI_COMMON - + [field("temperature", "Temperature", type_="number", default=0.7)], - }, { "interface_type": "dashscope-asr", "name": "DashScope ASR", @@ -165,7 +158,6 @@ INTERFACE_DEFINITIONS: list[dict] = [ }, ] - def validate_fields(definition: dict, values: dict, secrets: dict) -> None: for item in definition["fields"]: source = secrets if item["group"] == "secrets" else values diff --git a/backend/services/pipecat/pipeline.py b/backend/services/pipecat/pipeline.py index 5c37e7e..95b92e4 100644 --- a/backend/services/pipecat/pipeline.py +++ b/backend/services/pipecat/pipeline.py @@ -35,6 +35,7 @@ from pipecat.frames.frames import ( OutputTransportMessageUrgentFrame, TextFrame, TTSSpeakFrame, + UserImageRequestFrame, ) from pipecat.pipeline.pipeline import Pipeline from pipecat.pipeline.worker import PipelineParams, PipelineWorker @@ -45,6 +46,11 @@ from pipecat.processors.aggregators.llm_response_universal import ( LLMUserAggregatorParams, ) from pipecat.processors.frame_processor import FrameDirection, FrameProcessor +from pipecat.runner.utils import ( + get_transport_client_id, + maybe_capture_participant_camera, +) +from pipecat.services.llm_service import FunctionCallParams from pipecat.turns.user_start import ( TranscriptionUserTurnStartStrategy, VADUserTurnStartStrategy, @@ -54,6 +60,13 @@ from pipecat.utils.time import time_now_iso8601 from pipecat.workers.runner import WorkerRunner +VISION_TOOL_NAME = "fetch_user_image" +VISION_SYSTEM_HINT = ( + "当前会话打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/" + "环境状态或需要你看一眼时,调用 fetch_user_image 获取当前视频帧,再基于画面回答。" +) + + def _text_input(message) -> tuple[str, bool] | None: """解析现有 user-text 与 RTVI send-text 两种前端文字消息。""" if not isinstance(message, dict): @@ -204,7 +217,12 @@ class PassthroughLLMAssistantAggregator(LLMAssistantAggregator): self._stream_text = "" -async def run_pipeline(transport, cfg: AssistantConfig) -> None: +async def run_pipeline( + transport, + cfg: AssistantConfig, + *, + vision_enabled: bool = False, +) -> None: """在给定 transport 上构建并运行管线,直到连接结束。 Args: @@ -212,7 +230,10 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: 只要有 .input() / .output() / event_handler 即可。 cfg: 助手配置(随请求内联传入)。 """ - logger.info(f"启动管线: assistant={cfg.name} type={cfg.type} mode={cfg.runtimeMode}") + logger.info( + f"启动管线: assistant={cfg.name} type={cfg.type} " + f"mode={cfg.runtimeMode} vision={vision_enabled}" + ) # 大脑:按类型决定 LLM 槽/开场白/上下文归属。每通电话一个实例(可持会话状态)。 brain = build_brain(cfg) @@ -224,6 +245,8 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: cfg.runtimeMode = "pipeline" if cfg.runtimeMode == "realtime": + if vision_enabled: + logger.warning("Realtime 模式暂未接入视频帧工具,本次仅启用语音通话") await run_realtime_pipeline(transport, cfg) return @@ -263,7 +286,16 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: greeting = await brain.greeting(cfg) system_content = "" - context = LLMContext(messages=[{"role": "system", "content": system_content}]) + def with_vision_hint(text: str) -> str: + if not vision_enabled: + return text + if not text: + return VISION_SYSTEM_HINT + return f"{text}\n\n{VISION_SYSTEM_HINT}" + + context = LLMContext( + messages=[{"role": "system", "content": with_vision_hint(system_content)}] + ) # LLM 槽由大脑提供:内部类型=OpenAI 兼容服务;fastgpt=包 SDK 的伪 LLM。 llm = brain.build_llm(cfg, context) user_aggregator = LLMUserAggregator( @@ -282,6 +314,58 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: ) assistant_aggregator = PassthroughLLMAssistantAggregator(context) text_input = TextInputProcessor() + vision_state: dict[str, str | None] = {"client_id": None} + vision_schema = FunctionSchema( + name=VISION_TOOL_NAME, + description=( + "获取用户当前摄像头画面。当用户询问当前画面、看到了什么、" + "人/物品/环境状态或需要视觉判断时调用。" + ), + properties={ + "question": { + "type": "string", + "description": "用户关于当前视频画面的具体问题。", + } + }, + required=["question"], + ) + + async def fetch_user_image(params: FunctionCallParams): + question = str(params.arguments.get("question") or "请描述当前画面。") + user_id = vision_state.get("client_id") + if not user_id: + await params.result_callback( + { + "status": "no_video_client", + "message": "当前还没有可用的摄像头视频流。", + } + ) + return + + logger.debug(f"请求当前视频帧: user_id={user_id}, question={question}") + await params.llm.push_frame( + UserImageRequestFrame( + user_id=user_id, + text=question, + append_to_context=True, + function_name=params.function_name, + tool_call_id=params.tool_call_id, + result_callback=params.result_callback, + ), + FrameDirection.UPSTREAM, + ) + + if vision_enabled: + llm.register_function(VISION_TOOL_NAME, fetch_user_image) + + def set_visible_tools(schemas: list[FunctionSchema] | None = None) -> None: + tools = list(schemas or []) + if vision_enabled: + tools.append(vision_schema) + if tools: + context.set_tools(ToolsSchema(standard_tools=tools)) + else: + context.set_tools() # 结束节点:等结束语「说完」(BotStoppedSpeakingFrame)再挂断,确保结束语的 # 文字(走 data channel)与音频都已下发,避免前端只听到声音、看不到文字。 @@ -366,16 +450,17 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: def set_system_prompt(text: str) -> None: """替换上下文里的系统提示(节点切换时整体替换,而非追加)。""" messages = context.get_messages() + content = with_vision_hint(text) if messages and messages[0].get("role") == "system": - messages[0] = {"role": "system", "content": text} + messages[0] = {"role": "system", "content": content} else: - messages.insert(0, {"role": "system", "content": text}) + messages.insert(0, {"role": "system", "content": content}) def apply_node(node_id: str | None) -> None: """进入节点:设置系统提示 + 把出边注册为可调用的转移工具。""" set_system_prompt(engine.system_prompt_for(node_id)) if engine.is_end(node_id): - context.set_tools() # 终止节点无工具 + set_visible_tools([]) # 终止节点不展示转移工具,但保留视觉工具 return schemas = [ FunctionSchema( @@ -386,10 +471,7 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: ) for edge in engine.outgoing(node_id) ] - if schemas: - context.set_tools(ToolsSchema(standard_tools=schemas)) - else: - context.set_tools() # 无出边:清空工具 + set_visible_tools(schemas) async def go_to_node(target: str) -> None: """执行转移:切当前节点、重置计数、点亮画布、设置提示/工具。 @@ -455,6 +537,8 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: engine.edge_fn_name(edge), make_transition_handler(edge) ) apply_node(wf_state["current"]) # 设初始节点的提示与工具 + elif vision_enabled: + set_visible_tools([]) async def append_user_text_to_context(text: str, *, run_llm: bool) -> None: await worker.queue_frame( @@ -559,6 +643,16 @@ async def run_pipeline(transport, cfg: AssistantConfig) -> None: @transport.event_handler("on_client_connected") async def on_client_connected(_transport, _client): + if vision_enabled: + try: + vision_state["client_id"] = get_transport_client_id( + _transport, + _client, + ) + await maybe_capture_participant_camera(_transport, _client) + logger.info(f"视觉理解已接入视频客户端: {vision_state['client_id']}") + except Exception as e: + logger.warning(f"视觉理解摄像头捕获初始化失败: {e}") if greeting: # 外部托管类型的上下文由对方服务端维护,开场白不写入本地 context if brain.spec.owns_context: diff --git a/backend/services/pipecat/service_factory.py b/backend/services/pipecat/service_factory.py index 227e3bf..89317fd 100644 --- a/backend/services/pipecat/service_factory.py +++ b/backend/services/pipecat/service_factory.py @@ -72,10 +72,15 @@ def create_llm(cfg: AssistantConfig): """DeepSeek 等,走 OpenAI 兼容的 /v1/chat/completions。""" if cfg.llm_interface_type not in {"openai-llm", "dashscope-llm"}: raise ValueError(f"不支持的 LLM 接口类型: {cfg.llm_interface_type}") + extra_body = cfg.llm_values.get("extraBody") + extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {} return OpenAILLMService( api_key=cfg.llm_api_key or config.LLM_API_KEY, base_url=cfg.llm_base_url or config.LLM_BASE_URL, - settings=OpenAILLMService.Settings(model=cfg.model or config.LLM_MODEL), + settings=OpenAILLMService.Settings( + model=cfg.model or config.LLM_MODEL, + extra=extra, + ), ) diff --git a/backend/services/pipecat/transports.py b/backend/services/pipecat/transports.py index c7e8037..3a07471 100644 --- a/backend/services/pipecat/transports.py +++ b/backend/services/pipecat/transports.py @@ -24,21 +24,26 @@ from pipecat.transports.websocket.fastapi import ( from pipecat.serializers.protobuf import ProtobufFrameSerializer -def _base_params() -> dict: +def _base_params(*, video_in_enabled: bool = False) -> dict: """两种 transport 共享的音频参数。""" return dict( audio_in_enabled=True, audio_out_enabled=True, + video_in_enabled=video_in_enabled, # EndFrame 后默认补 2s 静音(防止收尾被截断)。我们的挂断已等到机器人 # 说完才触发,这段静音纯属空等,置 0 让结束语播完立即挂断。 audio_out_end_silence_secs=0, ) -def build_webrtc_transport(connection: SmallWebRTCConnection) -> SmallWebRTCTransport: +def build_webrtc_transport( + connection: SmallWebRTCConnection, + *, + video_in_enabled: bool = False, +) -> SmallWebRTCTransport: return SmallWebRTCTransport( webrtc_connection=connection, - params=TransportParams(**_base_params()), + params=TransportParams(**_base_params(video_in_enabled=video_in_enabled)), ) diff --git a/frontend/src/components/pages/AssistantPage.tsx b/frontend/src/components/pages/AssistantPage.tsx index 40ffcc7..494863a 100644 --- a/frontend/src/components/pages/AssistantPage.tsx +++ b/frontend/src/components/pages/AssistantPage.tsx @@ -426,10 +426,35 @@ export function AssistantPage(props: AssistantPageProps) { // 按资源类型生成 {value:id, label:name} 选项 const credOptions = (type: ModelResource["capability"]) => modelResources - .filter((c) => c.capability === type) + .filter((c) => { + if (c.capability !== type) return false; + if (type === "LLM" && visionEnabled) return c.supportImageInput; + return true; + }) .map((c) => ({ value: c.id, label: c.name })); const kbOptions = knowledgeBases.map((k) => ({ value: k.id, label: k.name })); + function modelSupportsImageInput(resourceId: string): boolean { + return Boolean( + modelResources.find((resource) => resource.id === resourceId) + ?.supportImageInput, + ); + } + + function handleVisionEnabledChange(enabled: boolean) { + setVisionEnabled(enabled); + if (enabled && form.model && !modelSupportsImageInput(form.model)) { + updateForm("model", ""); + } + if ( + enabled && + openCodeForm.model && + !modelSupportsImageInput(openCodeForm.model) + ) { + updateOpenCodeForm("model", ""); + } + } + function startCreate() { router.push("/assistants/new"); } @@ -1369,7 +1394,7 @@ export function AssistantPage(props: AssistantPageProps) { > updateDifyForm("enableInterrupt", checked) @@ -1483,7 +1508,7 @@ export function AssistantPage(props: AssistantPageProps) { > updateFastGptForm("enableInterrupt", checked) @@ -1572,6 +1597,12 @@ export function AssistantPage(props: AssistantPageProps) { title="模型与语音配置" description="配置 OpenCode 使用的大语言模型、语音识别与语音合成资源。" > + updateOpenCodeForm("enableInterrupt", checked) @@ -1724,15 +1755,6 @@ export function AssistantPage(props: AssistantPageProps) { -
- } - title="视觉理解" - hint="开启后，右侧调试面板可切换到视频流预览，并可同时选择麦克风与摄像头。" - checked={visionEnabled} - onChange={setVisionEnabled} - /> -
+ updateForm("enableInterrupt", checked)} /> @@ -1839,7 +1867,7 @@ export function AssistantPage(props: AssistantPageProps) { type VizStyle = "aura" | "nebula" | "bars" | "wave"; -// 调试面板顶部主视图:聊天记录 / 视频流(仅视觉理解开启时可选) +// 调试面板顶部主视图:聊天记录 / 视频流 type DebugView = "chat" | "video"; type DebugInputMode = "mic" | "text"; @@ -1919,28 +1947,27 @@ function DebugDrawer({ const [showTranscript, setShowTranscript] = useState(false); const [vizStyle, setVizStyle] = useState("aura"); const [view, setView] = useState("chat"); - const effectiveView: DebugView = vision ? view : "chat"; const stopCamera = camera.stop; const startCamera = camera.start; - const cameraShouldRun = - vision && - (preview.status === "connecting" || preview.status === "connected"); + const recording = + preview.status === "connecting" || preview.status === "connected"; + const shouldKeepCamera = vision && recording; useEffect(() => { - if (!vision) { - // eslint-disable-next-line react-hooks/set-state-in-effect - setView("chat"); - stopCamera(); - } - }, [vision, stopCamera]); - - useEffect(() => { - if (cameraShouldRun) { + if (shouldKeepCamera) { void startCamera(); } else { stopCamera(); } - }, [cameraShouldRun, camera.deviceId, startCamera, stopCamera]); + }, [shouldKeepCamera, startCamera, stopCamera]); + + const selectCamera = useCallback( + async (deviceId: string) => { + const nextStream = await camera.selectCamera(deviceId); + await preview.replaceVideoStream(nextStream); + }, + [camera, preview], + ); const containerClass = asSheet ? "flex h-full min-w-0 flex-1 flex-col overflow-hidden" @@ -1959,7 +1986,7 @@ function DebugDrawer({ />
- {SHOW_VOICE_VIZ && effectiveView === "chat" && ( + {SHOW_VOICE_VIZ && view === "chat" && ( <> {!showTranscript && ( @@ -1995,33 +2022,29 @@ function DebugDrawer({ )} setView("chat")} > - {vision && ( - setView("video")} - > - - )} + setView("video")} + > +
- {vision && ( -
-
- -
+
+
+
- )} +
void | Promise; +}) { return ( } @@ -2109,7 +2138,7 @@ function CameraDeviceField({ camera }: { camera: CameraPreview }) { fallbackLabel="摄像头" value={camera.deviceId} devices={camera.devices} - onSelect={(deviceId) => void camera.selectCamera(deviceId)} + onSelect={(deviceId) => void (onSelect ?? camera.selectCamera)(deviceId)} /> ); } @@ -2179,9 +2208,8 @@ function DebugVoicePanel({ const inChatView = view === "chat" && (!SHOW_VOICE_VIZ || showTranscript); const idleOrFailed = status === "idle" || status === "failed"; const showIdleHub = - messages.length === 0 && idleOrFailed && - (inChatView || (vision && view === "video")); + (view === "video" || (messages.length === 0 && inChatView)); function handleSendText() { if (sendText(textDraft)) { @@ -2189,26 +2217,36 @@ function DebugVoicePanel({ } } + const startConversation = useCallback(async () => { + const videoStream = vision ? await camera.start() : null; + await connect({ + visionEnabled: vision, + videoStream, + }); + }, [camera, connect, vision]); + return (
{/* 后端 TTS 音频经 WebRTC 媒体流过来,挂这里播放 */}