diff --git a/README.md b/README.md index 165b869..a7eca91 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,25 @@ Send audio: The adapter also accepts raw binary websocket messages as PCM16 audio chunks. JSON/base64 is easier to inspect; binary is better for latency and bandwidth. +Send a camera snapshot for vision-capable LLM replies: + +```json +{ + "type": "input.image", + "image": "", + "mime_type": "image/jpeg", + "width": 640, + "height": 360, + "text": "Answer using this camera image.", + "append_to_context": true +} +``` + +`input.image` appends the image to the Pipecat LLM context as a +`UserImageRawFrame` and immediately triggers the LLM. The reply returns through +the existing `response.text.*` and `response.audio.*` events. Prefer occasional +compressed camera snapshots over continuous video frames. + Stop: ```json diff --git a/docs/product-ws.md b/docs/product-ws.md index fd0fd1e..b93c704 100644 --- a/docs/product-ws.md +++ b/docs/product-ws.md @@ -69,13 +69,43 @@ ws://:/ws-product `interrupt` 默认为 `true`，会打断当前正在播放的 bot 语音。 -### 4. 取消当前回复 +### 4. 发送摄像头图片（可选） + +客户端可发送一帧摄像头截图，让 LLM 基于图片内容回复。服务端会把图片追加到 Pipecat LLM context，并立即触发一次 LLM 推理；后续回复仍通过现有的 `response.text.*` 与 `response.audio.*` 事件返回。 + +```json +{ + "type": "input.image", + "image": "", + "mime_type": "image/jpeg", + "width": 640, + "height": 360, + "text": "请根据这张摄像头画面回答用户的问题。", + "user_id": "product-user", + "append_to_context": true +} +``` + +字段说明： + +| 字段 | 说明 | +|------|------| +| `image` / `data` | 必填，图片字节的 base64；也兼容 `data:image/...;base64,...` data URL | +| `mime_type` / `media_type` | 可选，默认 `image/jpeg`；支持 `image/jpeg`、`image/png`、`image/webp` | +| `width`、`height` | 必填，图片像素尺寸，必须为正整数 | +| `text` | 可选，随图片一起进入 LLM 的问题/提示词 | +| `user_id` | 可选，默认 `product-user` | +| `append_to_context` | 可选，默认 `true`；为 `true` 时图片会进入 LLM context 并触发回复 | + +建议发送压缩后的截图而不是持续视频流，例如 640px 宽、JPEG quality 0.75 左右。单张图片最大 8 MB。 + +### 5. 取消当前回复 ```json {"type": "response.cancel"} ``` -### 5. 结束会话 +### 6. 结束会话 ```json {"type": "session.stop", "reason": "done"} diff --git a/engine/main.py b/engine/main.py index 64180f0..2e8dd95 100644 --- a/engine/main.py +++ b/engine/main.py @@ -56,6 +56,7 @@ def create_app(config_path: str = "config.json") -> FastAPI: "features": { "product_text_input": True, "product_text_interrupt": True, + "product_image_input": True, }, "demo": webpage_mount, "llm_provider": config.services.llm.provider, diff --git a/engine/product_protocol.py b/engine/product_protocol.py index 6f961ac..79c71b0 100644 --- a/engine/product_protocol.py +++ b/engine/product_protocol.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import binascii import json from typing import Any @@ -19,10 +20,15 @@ from pipecat.frames.frames import ( OutputTransportMessageUrgentFrame, TextFrame, TranscriptionFrame, + UserImageRawFrame, ) from pipecat.serializers.base_serializer import FrameSerializer +MAX_INPUT_IMAGE_BYTES = 8 * 1024 * 1024 +SUPPORTED_INPUT_IMAGE_MIME_TYPES = {"image/jpeg", "image/png", "image/webp"} + + class ProductWebsocketSerializer(FrameSerializer): """Stable app-facing JSON/base64 protocol adapter for Pipecat websocket transport.""" @@ -122,7 +128,7 @@ class ProductWebsocketSerializer(FrameSerializer): return None try: pcm = base64.b64decode(audio) - except ValueError as exc: + except (binascii.Error, ValueError) as exc: logger.warning(f"Invalid input.audio base64: {exc}") return None return InputAudioRawFrame( @@ -131,6 +137,9 @@ class ProductWebsocketSerializer(FrameSerializer): num_channels=int(message.get("channels") or self._channels), ) + if message_type == "input.image": + return self._deserialize_input_image(message) + if message_type == "input.text": text = message.get("text") if not isinstance(text, str) or not text.strip(): @@ -151,6 +160,61 @@ class ProductWebsocketSerializer(FrameSerializer): logger.warning(f"Unsupported product websocket message type: {message_type!r}") return None + def _deserialize_input_image(self, message: dict[str, Any]) -> Frame | None: + encoded = message.get("image") or message.get("data") + if not isinstance(encoded, str): + logger.warning("input.image requires base64 'image' or 'data'") + return None + + mime_type = str(message.get("mime_type") or message.get("media_type") or "image/jpeg") + if mime_type not in SUPPORTED_INPUT_IMAGE_MIME_TYPES: + logger.warning( + "input.image unsupported mime_type " + f"{mime_type!r}; expected one of {sorted(SUPPORTED_INPUT_IMAGE_MIME_TYPES)}" + ) + return None + + try: + width = int(message.get("width") or 0) + height = int(message.get("height") or 0) + except (TypeError, ValueError): + logger.warning("input.image width and height must be integers") + return None + + if width <= 0 or height <= 0: + logger.warning("input.image requires positive integer width and height") + return None + + if "," in encoded and encoded.lstrip().startswith("data:"): + encoded = encoded.split(",", 1)[1] + + try: + image = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as exc: + logger.warning(f"Invalid input.image base64: {exc}") + return None + + if len(image) > MAX_INPUT_IMAGE_BYTES: + logger.warning( + f"input.image too large: {len(image)} bytes; " + f"max is {MAX_INPUT_IMAGE_BYTES} bytes" + ) + return None + + text = message.get("text") + if text is not None and not isinstance(text, str): + logger.warning("input.image text must be a string when provided") + return None + + return UserImageRawFrame( + image=image, + size=(width, height), + format=mime_type, + user_id=str(message.get("user_id") or "product-user"), + text=text or "Answer using this camera image.", + append_to_context=bool(message.get("append_to_context", True)), + ) + def _event(self, event_type: str, **payload: Any) -> str: self._sequence += 1 return json.dumps(