Add vision model enhancements and image processing capabilities
- Update `requirements.txt` to include Pillow for image handling. - Refactor vision model validation logic in `voice_webrtc.py` to improve error handling for unsupported image input. - Introduce new functions in `pipeline.py` for image data processing and analysis using vision models. - Implement `VisionCaptureProcessor` to manage video frame requests for auxiliary vision model analysis. - Enhance the pipeline to support image input requests and integrate vision model responses into the processing flow.
This commit is contained in:
@@ -74,16 +74,6 @@ async def _resolve_config(offer: SignalingOffer) -> AssistantConfig:
|
||||
raise ValueError("offer 缺少 assistant_id 或 inline_config")
|
||||
|
||||
|
||||
def _apply_vision_model(cfg: AssistantConfig) -> None:
|
||||
cfg.model = cfg.vision_model
|
||||
cfg.llm_interface_type = cfg.vision_llm_interface_type
|
||||
cfg.llm_values = cfg.vision_llm_values
|
||||
cfg.llm_secrets = cfg.vision_llm_secrets
|
||||
cfg.llm_support_image_input = cfg.vision_llm_support_image_input
|
||||
cfg.llm_api_key = cfg.vision_llm_api_key
|
||||
cfg.llm_base_url = cfg.vision_llm_base_url
|
||||
|
||||
|
||||
async def _handle_offer(websocket, payload, peers):
|
||||
from pipecat.transports.smallwebrtc.connection import SmallWebRTCConnection
|
||||
from services.pipecat.pipeline import run_pipeline
|
||||
@@ -99,11 +89,17 @@ async def _handle_offer(websocket, payload, peers):
|
||||
cfg = await _resolve_config(offer) # 解析放在建连前,配置错就别建连
|
||||
vision_enabled = offer.vision_enabled or cfg.vision_enabled
|
||||
if vision_enabled:
|
||||
if not cfg.vision_llm_support_image_input:
|
||||
has_native_vision = (
|
||||
not cfg.vision_model_resource_id and cfg.llm_support_image_input
|
||||
)
|
||||
has_aux_vision_model = (
|
||||
bool(cfg.vision_model_resource_id)
|
||||
and cfg.vision_llm_support_image_input
|
||||
)
|
||||
if not (has_native_vision or has_aux_vision_model):
|
||||
raise ValueError(
|
||||
"当前视觉模型不支持图片输入,请在模型资源中选择支持图片输入的 LLM"
|
||||
"当前模型不支持图片输入,请在模型资源中选择支持图片输入的视觉模型"
|
||||
)
|
||||
_apply_vision_model(cfg)
|
||||
pc = SmallWebRTCConnection(ice_servers=aiortc_ice_servers())
|
||||
if pc_id:
|
||||
pc._pc_id = pc_id
|
||||
|
||||
Reference in New Issue
Block a user