Add vision model enhancements and image processing capabilities

- Update `requirements.txt` to include Pillow for image handling.
- Refactor vision model validation logic in `voice_webrtc.py` to improve error handling for unsupported image input.
- Introduce new functions in `pipeline.py` for image data processing and analysis using vision models.
- Implement `VisionCaptureProcessor` to manage video frame requests for auxiliary vision model analysis.
- Enhance the pipeline to support image input requests and integrate vision model responses into the processing flow.
This commit is contained in:
Xin Wang
2026-07-08 10:33:44 +08:00
parent b428f1b8cf
commit 5bc4e24adb
3 changed files with 169 additions and 24 deletions

View File

@@ -74,16 +74,6 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
raise ValueError("offer 缺少 assistant_id 或 inline_config")
def _apply_vision_model(cfg: AssistantConfig) -> None:
cfg.model = cfg.vision_model
cfg.llm_interface_type = cfg.vision_llm_interface_type
cfg.llm_values = cfg.vision_llm_values
cfg.llm_secrets = cfg.vision_llm_secrets
cfg.llm_support_image_input = cfg.vision_llm_support_image_input
cfg.llm_api_key = cfg.vision_llm_api_key
cfg.llm_base_url = cfg.vision_llm_base_url
async def _handle_offer(websocket, payload, peers):
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
from services.pipecat.pipeline import run_pipeline
@@ -99,11 +89,17 @@ async def _handle_offer(websocket, payload, peers):
cfg = await _resolve_config(offer) # 解析放在建连前,配置错就别建连
vision_enabled = offer.vision_enabled or cfg.vision_enabled
if vision_enabled:
if not cfg.vision_llm_support_image_input:
has_native_vision = (
not cfg.vision_model_resource_id and cfg.llm_support_image_input
)
has_aux_vision_model = (
bool(cfg.vision_model_resource_id)
and cfg.vision_llm_support_image_input
)
if not (has_native_vision or has_aux_vision_model):
raise ValueError(
"当前视觉模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM"
"当前模型不支持图片输入,请在模型资源中选择支持图片输入的视觉模型"
)
_apply_vision_model(cfg)
pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
if pc_id:
pc._pc_id = pc_id