Add input image to product protocol

engine/main.py

@@ -56,6 +56,7 @@ def create_app(config_path: str = "config.json") -> FastAPI:
             "features": {
                 "product_text_input": True,
                 "product_text_interrupt": True,
+                "product_image_input": True,
             },
             "demo": webpage_mount,
             "llm_provider": config.services.llm.provider,

engine/product_protocol.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import base64
+import binascii
 import json
 from typing import Any
 
@@ -19,10 +20,15 @@ from pipecat.frames.frames import (
     OutputTransportMessageUrgentFrame,
     TextFrame,
     TranscriptionFrame,
+    UserImageRawFrame,
 )
 from pipecat.serializers.base_serializer import FrameSerializer
 
 
+MAX_INPUT_IMAGE_BYTES = 8 * 1024 * 1024
+SUPPORTED_INPUT_IMAGE_MIME_TYPES = {"image/jpeg", "image/png", "image/webp"}
+
+
 class ProductWebsocketSerializer(FrameSerializer):
     """Stable app-facing JSON/base64 protocol adapter for Pipecat websocket transport."""
 
@@ -122,7 +128,7 @@ class ProductWebsocketSerializer(FrameSerializer):
                 return None
             try:
                 pcm = base64.b64decode(audio)
-            except ValueError as exc:
+            except (binascii.Error, ValueError) as exc:
                 logger.warning(f"Invalid input.audio base64: {exc}")
                 return None
             return InputAudioRawFrame(
@@ -131,6 +137,9 @@ class ProductWebsocketSerializer(FrameSerializer):
                 num_channels=int(message.get("channels") or self._channels),
             )
 
+        if message_type == "input.image":
+            return self._deserialize_input_image(message)
+
         if message_type == "input.text":
             text = message.get("text")
             if not isinstance(text, str) or not text.strip():
@@ -151,6 +160,61 @@ class ProductWebsocketSerializer(FrameSerializer):
         logger.warning(f"Unsupported product websocket message type: {message_type!r}")
         return None
 
+    def _deserialize_input_image(self, message: dict[str, Any]) -> Frame | None:
+        encoded = message.get("image") or message.get("data")
+        if not isinstance(encoded, str):
+            logger.warning("input.image requires base64 'image' or 'data'")
+            return None
+
+        mime_type = str(message.get("mime_type") or message.get("media_type") or "image/jpeg")
+        if mime_type not in SUPPORTED_INPUT_IMAGE_MIME_TYPES:
+            logger.warning(
+                "input.image unsupported mime_type "
+                f"{mime_type!r}; expected one of {sorted(SUPPORTED_INPUT_IMAGE_MIME_TYPES)}"
+            )
+            return None
+
+        try:
+            width = int(message.get("width") or 0)
+            height = int(message.get("height") or 0)
+        except (TypeError, ValueError):
+            logger.warning("input.image width and height must be integers")
+            return None
+
+        if width <= 0 or height <= 0:
+            logger.warning("input.image requires positive integer width and height")
+            return None
+
+        if "," in encoded and encoded.lstrip().startswith("data:"):
+            encoded = encoded.split(",", 1)[1]
+
+        try:
+            image = base64.b64decode(encoded, validate=True)
+        except (binascii.Error, ValueError) as exc:
+            logger.warning(f"Invalid input.image base64: {exc}")
+            return None
+
+        if len(image) > MAX_INPUT_IMAGE_BYTES:
+            logger.warning(
+                f"input.image too large: {len(image)} bytes; "
+                f"max is {MAX_INPUT_IMAGE_BYTES} bytes"
+            )
+            return None
+
+        text = message.get("text")
+        if text is not None and not isinstance(text, str):
+            logger.warning("input.image text must be a string when provided")
+            return None
+
+        return UserImageRawFrame(
+            image=image,
+            size=(width, height),
+            format=mime_type,
+            user_id=str(message.get("user_id") or "product-user"),
+            text=text or "Answer using this camera image.",
+            append_to_context=bool(message.get("append_to_context", True)),
+        )
+
     def _event(self, event_type: str, **payload: Any) -> str:
         self._sequence += 1
         return json.dumps(