ai-video-fullstack/backend/services/vision.py

"""Shared camera-frame analysis for Pipeline and Realtime runtimes."""

from __future__ import annotations

import asyncio
import base64
from io import BytesIO

from models import AssistantConfig, RuntimeModelResource
from openai import AsyncOpenAI
from PIL import Image
from pipecat.frames.frames import UserImageRawFrame


VISION_TOOL_NAME = "fetch_user_image"
VISION_SYSTEM_HINT = (
    "当前阶段打开了视觉理解。用户询问当前画面、摄像头里有什么、人物/物品/"
    "环境状态或需要你看一眼时，调用 fetch_user_image 获取当前视频帧，"
    "再基于画面回答。"
)
VISION_ANALYSIS_SYSTEM_PROMPT = (
    "你是一个视觉理解模型。请只根据图片内容和用户问题给出准确、简洁的中文观察结果。"
    "如果画面不足以判断，请明确说明不确定。"
)


def _require(value: str, label: str) -> str:
    if value:
        return value
    raise ValueError(f"缺少模型资源配置: {label}")


def image_jpeg_bytes(frame: UserImageRawFrame) -> bytes:
    """Encode one Pipecat camera frame as a storage- and model-ready JPEG."""
    if not frame.format:
        raise ValueError("摄像头图片帧缺少 format,无法编码给视觉模型")
    buffer = BytesIO()
    Image.frombytes(frame.format, frame.size, frame.image).save(
        buffer,
        format="JPEG",
        quality=85,
    )
    return buffer.getvalue()


def image_data_uri(frame: UserImageRawFrame) -> str:
    """Encode one Pipecat camera frame for a vision chat-completion request."""
    encoded = base64.b64encode(image_jpeg_bytes(frame)).decode("utf-8")
    return f"data:image/jpeg;base64,{encoded}"


def image_frame_from_jpeg(data: bytes) -> UserImageRawFrame:
    """Decode one normalized uploaded JPEG into Pipecat's raw image frame."""

    try:
        image = Image.open(BytesIO(data)).convert("RGB")
        image.load()
    except (OSError, ValueError) as exc:
        raise ValueError("上传图片无法解码") from exc
    return UserImageRawFrame(
        image=image.tobytes(),
        size=image.size,
        format="RGB",
    )


async def analyze_image_with_vision_model(
    cfg: AssistantConfig,
    frame: UserImageRawFrame,
    question: str,
) -> str:
    """Analyze one frame with the configured independent vision LLM."""
    if cfg.vision_llm_interface_type not in {"openai-llm", "dashscope-llm"}:
        raise ValueError(
            f"不支持的视觉 LLM 接口类型: {cfg.vision_llm_interface_type}"
        )

    data_uri = await asyncio.to_thread(image_data_uri, frame)
    extra_body = cfg.vision_llm_values.get("extraBody")
    extra = {"extra_body": extra_body} if isinstance(extra_body, dict) else {}
    client = AsyncOpenAI(
        api_key=_require(cfg.vision_llm_api_key, "Vision LLM apiKey"),
        base_url=_require(cfg.vision_llm_base_url, "Vision LLM apiUrl"),
    )
    try:
        response = await client.chat.completions.create(
            model=_require(cfg.vision_model, "Vision LLM modelId"),
            messages=[
                {"role": "system", "content": VISION_ANALYSIS_SYSTEM_PROMPT},
                {
                    "role": "user",
                    "content": [
                        {"type": "text", "text": question},
                        {"type": "image_url", "image_url": {"url": data_uri}},
                    ],
                },
            ],
            **extra,
        )
    finally:
        await client.close()

    content = response.choices[0].message.content if response.choices else ""
    if isinstance(content, str):
        return content.strip()
    return str(content or "").strip()


def config_with_vision_resource(
    cfg: AssistantConfig,
    resource: RuntimeModelResource,
) -> AssistantConfig:
    """Create the config view used to analyze one Workflow camera frame."""
    if resource.capability != "LLM":
        raise ValueError(f"视觉模型资源能力无效:{resource.capability}")
    if not resource.support_image_input:
        raise ValueError(f"视觉模型不支持图片输入:{resource.id}")

    result = cfg.model_copy(deep=True)
    values = resource.values or {}
    secrets = resource.secrets or {}
    result.vision_model_resource_id = resource.id
    result.vision_model = str(values.get("modelId") or "")
    result.vision_llm_interface_type = resource.interface_type
    result.vision_llm_values = values
    result.vision_llm_secrets = secrets
    result.vision_llm_support_image_input = resource.support_image_input
    result.vision_llm_api_key = str(secrets.get("apiKey") or "")
    result.vision_llm_base_url = str(values.get("apiUrl") or "")
    return result


def config_with_main_llm_as_vision(cfg: AssistantConfig) -> AssistantConfig:
    """Legacy fallback when a Workflow LLM is not present in the resource map."""
    if not cfg.llm_support_image_input:
        raise ValueError("当前大语言模型不支持图片输入")

    result = cfg.model_copy(deep=True)
    result.vision_model_resource_id = None
    result.vision_model = cfg.model
    result.vision_llm_interface_type = cfg.llm_interface_type
    result.vision_llm_values = dict(cfg.llm_values)
    result.vision_llm_secrets = dict(cfg.llm_secrets)
    result.vision_llm_support_image_input = cfg.llm_support_image_input
    result.vision_llm_api_key = cfg.llm_api_key
    result.vision_llm_base_url = cfg.llm_base_url
    return result