Add input image to product protocol

README.md

@@ -95,6 +95,25 @@ Send audio:
 
 The adapter also accepts raw binary websocket messages as PCM16 audio chunks. JSON/base64 is easier to inspect; binary is better for latency and bandwidth.
 
+Send a camera snapshot for vision-capable LLM replies:
+
+```json
+{
+  "type": "input.image",
+  "image": "<base64 jpeg/png/webp bytes>",
+  "mime_type": "image/jpeg",
+  "width": 640,
+  "height": 360,
+  "text": "Answer using this camera image.",
+  "append_to_context": true
+}
+```
+
+`input.image` appends the image to the Pipecat LLM context as a
+`UserImageRawFrame` and immediately triggers the LLM. The reply returns through
+the existing `response.text.*` and `response.audio.*` events. Prefer occasional
+compressed camera snapshots over continuous video frames.
+
 Stop:
 
 ```json

docs/product-ws.md

@@ -69,13 +69,43 @@ ws://<host>:<port>/ws-product
 
 `interrupt` 默认为 `true`，会打断当前正在播放的 bot 语音。
 
-### 4. 取消当前回复
+### 4. 发送摄像头图片（可选）
+
+客户端可发送一帧摄像头截图，让 LLM 基于图片内容回复。服务端会把图片追加到 Pipecat LLM context，并立即触发一次 LLM 推理；后续回复仍通过现有的 `response.text.*` 与 `response.audio.*` 事件返回。
+
+```json
+{
+  "type": "input.image",
+  "image": "<base64 编码的 JPEG/PNG/WebP 图片字节>",
+  "mime_type": "image/jpeg",
+  "width": 640,
+  "height": 360,
+  "text": "请根据这张摄像头画面回答用户的问题。",
+  "user_id": "product-user",
+  "append_to_context": true
+}
+```
+
+字段说明：
+
+| 字段 | 说明 |
+|------|------|
+| `image` / `data` | 必填，图片字节的 base64；也兼容 `data:image/...;base64,...` data URL |
+| `mime_type` / `media_type` | 可选，默认 `image/jpeg`；支持 `image/jpeg`、`image/png`、`image/webp` |
+| `width`、`height` | 必填，图片像素尺寸，必须为正整数 |
+| `text` | 可选，随图片一起进入 LLM 的问题/提示词 |
+| `user_id` | 可选，默认 `product-user` |
+| `append_to_context` | 可选，默认 `true`；为 `true` 时图片会进入 LLM context 并触发回复 |
+
+建议发送压缩后的截图而不是持续视频流，例如 640px 宽、JPEG quality 0.75 左右。单张图片最大 8 MB。
+
+### 5. 取消当前回复
 
 ```json
 {"type": "response.cancel"}
 ```
 
-### 5. 结束会话
+### 6. 结束会话
 
 ```json
 {"type": "session.stop", "reason": "done"}

engine/main.py

@@ -56,6 +56,7 @@ def create_app(config_path: str = "config.json") -> FastAPI:
             "features": {
                 "product_text_input": True,
                 "product_text_interrupt": True,
+                "product_image_input": True,
             },
             "demo": webpage_mount,
             "llm_provider": config.services.llm.provider,

engine/product_protocol.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import base64
+import binascii
 import json
 from typing import Any
 
@@ -19,10 +20,15 @@ from pipecat.frames.frames import (
     OutputTransportMessageUrgentFrame,
     TextFrame,
     TranscriptionFrame,
+    UserImageRawFrame,
 )
 from pipecat.serializers.base_serializer import FrameSerializer
 
 
+MAX_INPUT_IMAGE_BYTES = 8 * 1024 * 1024
+SUPPORTED_INPUT_IMAGE_MIME_TYPES = {"image/jpeg", "image/png", "image/webp"}
+
+
 class ProductWebsocketSerializer(FrameSerializer):
     """Stable app-facing JSON/base64 protocol adapter for Pipecat websocket transport."""
 
@@ -122,7 +128,7 @@ class ProductWebsocketSerializer(FrameSerializer):
                 return None
             try:
                 pcm = base64.b64decode(audio)
-            except ValueError as exc:
+            except (binascii.Error, ValueError) as exc:
                 logger.warning(f"Invalid input.audio base64: {exc}")
                 return None
             return InputAudioRawFrame(
@@ -131,6 +137,9 @@ class ProductWebsocketSerializer(FrameSerializer):
                 num_channels=int(message.get("channels") or self._channels),
             )
 
+        if message_type == "input.image":
+            return self._deserialize_input_image(message)
+
         if message_type == "input.text":
             text = message.get("text")
             if not isinstance(text, str) or not text.strip():
@@ -151,6 +160,61 @@ class ProductWebsocketSerializer(FrameSerializer):
         logger.warning(f"Unsupported product websocket message type: {message_type!r}")
         return None
 
+    def _deserialize_input_image(self, message: dict[str, Any]) -> Frame | None:
+        encoded = message.get("image") or message.get("data")
+        if not isinstance(encoded, str):
+            logger.warning("input.image requires base64 'image' or 'data'")
+            return None
+
+        mime_type = str(message.get("mime_type") or message.get("media_type") or "image/jpeg")
+        if mime_type not in SUPPORTED_INPUT_IMAGE_MIME_TYPES:
+            logger.warning(
+                "input.image unsupported mime_type "
+                f"{mime_type!r}; expected one of {sorted(SUPPORTED_INPUT_IMAGE_MIME_TYPES)}"
+            )
+            return None
+
+        try:
+            width = int(message.get("width") or 0)
+            height = int(message.get("height") or 0)
+        except (TypeError, ValueError):
+            logger.warning("input.image width and height must be integers")
+            return None
+
+        if width <= 0 or height <= 0:
+            logger.warning("input.image requires positive integer width and height")
+            return None
+
+        if "," in encoded and encoded.lstrip().startswith("data:"):
+            encoded = encoded.split(",", 1)[1]
+
+        try:
+            image = base64.b64decode(encoded, validate=True)
+        except (binascii.Error, ValueError) as exc:
+            logger.warning(f"Invalid input.image base64: {exc}")
+            return None
+
+        if len(image) > MAX_INPUT_IMAGE_BYTES:
+            logger.warning(
+                f"input.image too large: {len(image)} bytes; "
+                f"max is {MAX_INPUT_IMAGE_BYTES} bytes"
+            )
+            return None
+
+        text = message.get("text")
+        if text is not None and not isinstance(text, str):
+            logger.warning("input.image text must be a string when provided")
+            return None
+
+        return UserImageRawFrame(
+            image=image,
+            size=(width, height),
+            format=mime_type,
+            user_id=str(message.get("user_id") or "product-user"),
+            text=text or "Answer using this camera image.",
+            append_to_context=bool(message.get("append_to_context", True)),
+        )
+
     def _event(self, event_type: str, **payload: Any) -> str:
         self._sequence += 1
         return json.dumps(