Add image upload in conversation

engine/config.py

@@ -148,6 +148,8 @@ class LLMConfig:
     variables: dict[str, str] = field(default_factory=dict)
     detail: bool = False
     timeout_sec: float = 60.0
+    # FastGPT image input mode: "base64" (inline data URL) or "upload" (presigned upload).
+    image_input_mode: str = "base64"
 
     @property
     def is_fastgpt(self) -> bool:
@@ -257,6 +259,15 @@ def config_from_dict(data: dict) -> EngineConfig:
         llm["app_id"] = None
     if not isinstance(llm.get("variables"), dict):
         llm["variables"] = {}
+    image_input_mode = str(
+        llm.get("image_input_mode", LLMConfig().image_input_mode)
+    ).strip().lower()
+    if image_input_mode not in {"base64", "upload"}:
+        raise ValueError(
+            "services.llm.image_input_mode must be 'base64' or 'upload', "
+            f"got {llm.get('image_input_mode')!r}"
+        )
+    llm["image_input_mode"] = image_input_mode
     if agent.get("greeting_mode") == "fastgpt_opener" and llm["provider"] != "fastgpt":
         raise ValueError(
             "agent.greeting_mode='fastgpt_opener' requires services.llm.provider='fastgpt'"

engine/fastgpt_llm.py

@@ -1,7 +1,11 @@
 from __future__ import annotations
 
 import asyncio
+import base64
+import binascii
 import json
+import os
+import tempfile
 import uuid
 from dataclasses import dataclass, field
 from typing import Any
@@ -73,6 +77,50 @@ def _message_text(message: dict[str, Any]) -> str:
     return ""
 
 
+IMAGE_INPUT_MODE_BASE64 = "base64"
+IMAGE_INPUT_MODE_UPLOAD = "upload"
+SUPPORTED_IMAGE_INPUT_MODES = frozenset({IMAGE_INPUT_MODE_BASE64, IMAGE_INPUT_MODE_UPLOAD})
+
+_MIME_TO_EXT = {
+    "image/jpeg": ".jpg",
+    "image/png": ".png",
+    "image/webp": ".webp",
+}
+
+
+def _message_has_image(message: dict[str, Any]) -> bool:
+    content = message.get("content")
+    if not isinstance(content, list):
+        return False
+    return any(
+        isinstance(part, dict) and part.get("type") == "image_url"
+        for part in content
+    )
+
+
+def _redact_messages_for_log(messages: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Replace base64 image data URLs with a short placeholder for logging."""
+    redacted: list[dict[str, Any]] = []
+    for message in messages:
+        content = message.get("content")
+        if not isinstance(content, list):
+            redacted.append(message)
+            continue
+        parts: list[Any] = []
+        for part in content:
+            if (
+                isinstance(part, dict)
+                and part.get("type") == "image_url"
+                and isinstance(part.get("image_url"), dict)
+            ):
+                url = str(part["image_url"].get("url") or "")
+                parts.append({"type": "image_url", "image_url": {"url": f"<{len(url)} chars>"}})
+            else:
+                parts.append(part)
+        redacted.append({**message, "content": parts})
+    return redacted
+
+
 def _first_nonempty_text(*values: Any) -> str:
     for value in values:
         if isinstance(value, str):
@@ -172,6 +220,7 @@ class FastGPTLLMService(LLMService):
         app_id: str | None = None,
         greeting_prompt: str | None = None,
         timeout: float = 60.0,
+        image_input_mode: str = IMAGE_INPUT_MODE_BASE64,
         settings: FastGPTLLMSettings | None = None,
         **kwargs,
     ) -> None:
@@ -183,6 +232,20 @@ class FastGPTLLMService(LLMService):
         self._chat_id = chat_id or f"voice_{uuid.uuid4().hex[:16]}"
         self._app_id = (app_id or "").strip()
         self._greeting_prompt = (greeting_prompt or "你好").strip() or "你好"
+
+        mode = (image_input_mode or IMAGE_INPUT_MODE_BASE64).strip().lower()
+        if mode not in SUPPORTED_IMAGE_INPUT_MODES:
+            raise ValueError(
+                f"Unsupported image_input_mode {image_input_mode!r}; "
+                f"expected one of {sorted(SUPPORTED_IMAGE_INPUT_MODES)}"
+            )
+        if mode == IMAGE_INPUT_MODE_UPLOAD and not self._app_id:
+            logger.warning(
+                "FastGPT image_input_mode='upload' requires app_id; "
+                "falling back to inline base64"
+            )
+            mode = IMAGE_INPUT_MODE_BASE64
+        self._image_input_mode = mode
         self._client = AsyncChatClient(
             api_key=api_key,
             base_url=base_url,
@@ -310,26 +373,114 @@ class FastGPTLLMService(LLMService):
         if response is not None:
             await response.aclose()
 
-    def _build_fastgpt_messages(self, context: LLMContext) -> list[dict[str, str]]:
+    def _build_fastgpt_messages(self, context: LLMContext) -> list[dict[str, Any]]:
         raw_messages = context.get_messages()
 
         for message in reversed(raw_messages):
             if not isinstance(message, dict) or message.get("role") != "user":
                 continue
+            if _message_has_image(message):
+                # Multimodal turn: forward the OpenAI-style content list as-is
+                # (text parts + image_url with a base64 data URL). FastGPT's
+                # /chat/completions accepts this directly.
+                return [{"role": "user", "content": message["content"]}]
             text = _message_text(message)
             if text:
                 return [{"role": "user", "content": text}]
 
         return [{"role": "user", "content": self._greeting_prompt}]
 
+    async def _resolve_image_inputs(
+        self, messages: list[dict[str, Any]]
+    ) -> list[dict[str, Any]]:
+        """In ``upload`` mode, replace inline base64 image data URLs with uploaded URLs.
+
+        In ``base64`` mode the messages are returned untouched (inline data URLs).
+        New message/content objects are built so the shared ``LLMContext`` messages
+        are never mutated.
+        """
+        if self._image_input_mode != IMAGE_INPUT_MODE_UPLOAD:
+            return messages
+
+        resolved: list[dict[str, Any]] = []
+        for message in messages:
+            content = message.get("content")
+            if not isinstance(content, list):
+                resolved.append(message)
+                continue
+
+            new_content: list[Any] = []
+            for part in content:
+                url = (
+                    part.get("image_url", {}).get("url")
+                    if isinstance(part, dict) and part.get("type") == "image_url"
+                    else None
+                )
+                if isinstance(url, str) and url.startswith("data:image/"):
+                    uploaded = await self._upload_data_url(url)
+                    new_content.append(
+                        {"type": "image_url", "image_url": {"url": uploaded}}
+                    )
+                else:
+                    new_content.append(part)
+            resolved.append({**message, "content": new_content})
+
+        return resolved
+
+    async def _upload_data_url(self, data_url: str) -> str:
+        """Upload a ``data:image/...;base64,...`` URL via FastGPT and return its URL.
+
+        Falls back to the original data URL if parsing or upload fails so the turn
+        still proceeds with inline base64.
+        """
+        header, _, payload = data_url.partition(",")
+        mime_type = header[len("data:") :].split(";", 1)[0].strip() or "image/jpeg"
+        try:
+            raw = base64.b64decode(payload, validate=True)
+        except (binascii.Error, ValueError) as exc:
+            logger.warning(f"FastGPT image upload skipped; invalid base64: {exc}")
+            return data_url
+
+        suffix = _MIME_TO_EXT.get(mime_type, ".jpg")
+        tmp_path: str | None = None
+        try:
+            with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
+                tmp.write(raw)
+                tmp_path = tmp.name
+            result = await self._client.upload_chat_image(
+                appId=self._app_id,
+                chatId=self._chat_id,
+                file_path=tmp_path,
+            )
+            url = result.get("url") if isinstance(result, dict) else None
+            if isinstance(url, str) and url:
+                logger.info(
+                    f"FastGPT image uploaded chatId={self._chat_id} "
+                    f"bytes={len(raw)} url={url}"
+                )
+                return url
+            logger.warning("FastGPT image upload returned no url; using inline base64")
+            return data_url
+        except Exception as exc:
+            logger.warning(f"FastGPT image upload failed; using inline base64: {exc}")
+            return data_url
+        finally:
+            if tmp_path is not None:
+                try:
+                    os.unlink(tmp_path)
+                except OSError:
+                    pass
+
     async def _process_context(self, context: LLMContext) -> None:
         messages = self._build_fastgpt_messages(context)
+        messages = await self._resolve_image_inputs(messages)
         variables = self._settings.variables or None
 
         logger.info(
             "FastGPT chat completion "
             f"chatId={self._chat_id} appId={self._app_id or '-'} "
-            f"variables={sorted((variables or {}).keys())} messages={messages!r}"
+            f"variables={sorted((variables or {}).keys())} "
+            f"messages={_redact_messages_for_log(messages)!r}"
         )
 
         await self.start_ttfb_metrics()

engine/services.py

@@ -65,6 +65,7 @@ def create_llm_service(
             app_id=config.app_id,
             greeting_prompt=greeting_prompt,
             timeout=config.timeout_sec,
+            image_input_mode=config.image_input_mode,
             settings=FastGPTLLMSettings(
                 model=config.model or "fastgpt",
                 variables=variables,