Add image upload in conversation

examples/webpage/app.js

@@ -24,6 +24,17 @@ const WS_LOG_GROUP_KEYS = {
   AUDIO_SEND: "send:input.audio",
 };
 const CAMERA_DONE_TEXT = "【拍摄完成】";
+// Sample images shown as thumbnails under the camera preview. Same-origin files
+// so they can be drawn to a canvas (for base64 + dimensions) without tainting.
+const SAMPLE_IMAGES = [
+  { src: "./samples/front-damage.jpg", label: "车辆前部" },
+  { src: "./samples/plate.jpg", label: "车牌" },
+  { src: "./samples/license.jpg", label: "驾驶证" },
+  { src: "./samples/scene.jpg", label: "事故现场" },
+];
+// Cap the longer edge before JPEG-encoding so payloads stay small.
+const IMAGE_MAX_DIM = 1280;
+const IMAGE_JPEG_QUALITY = 0.85;
 const CAMERA_STATE_PROMPTS = {
   2000: "请对准车辆碰撞部位拍摄照片。",
   2001: "请对准车辆碰撞部位拍摄照片。",
@@ -62,6 +73,14 @@ const els = {
   cameraState: document.getElementById("camera-state"),
   cameraQuestion: document.getElementById("camera-question"),
   cameraDoneBtn: document.getElementById("camera-done-btn"),
+  cameraPreview: document.getElementById("camera-preview"),
+  cameraVideo: document.getElementById("camera-video"),
+  cameraPhoto: document.getElementById("camera-photo"),
+  cameraCanvas: document.getElementById("camera-canvas"),
+  cameraStartBtn: document.getElementById("camera-start-btn"),
+  cameraFlipBtn: document.getElementById("camera-flip-btn"),
+  cameraUpload: document.getElementById("camera-upload"),
+  cameraSamples: document.getElementById("camera-samples"),
   clearBtn: document.getElementById("clear-btn"),
   clearWsLogBtn: document.getElementById("clear-ws-log-btn"),
   wsLog: document.getElementById("ws-log"),
@@ -125,6 +144,13 @@ const state = {
   assistantState: "",
   cameraState: "",
 
+  // Camera / image input.
+  cameraStream: null,
+  cameraActive: false,
+  cameraFacing: "environment",
+  pendingImage: null,
+  samplesRendered: false,
+
   // VU meter smoothing.
   meterLevel: 0,
 
@@ -211,14 +237,16 @@ function setAssistantState(value) {
 
 function setCameraButtonEnabled() {
   if (!els.cameraDoneBtn) return;
-  els.cameraDoneBtn.disabled =
-    !state.connected || !state.cameraState ||
-    !state.ws || state.ws.readyState !== WebSocket.OPEN;
+  const wsReady =
+    state.connected && state.ws && state.ws.readyState === WebSocket.OPEN;
+  const hasImageSource = state.cameraActive || Boolean(state.pendingImage);
+  els.cameraDoneBtn.disabled = !wsReady || !state.cameraState || !hasImageSource;
 }
 
 function syncCameraDrawer(value) {
   const prompt = CAMERA_STATE_PROMPTS[value];
   const open = Boolean(prompt);
+  const wasOpen = Boolean(state.cameraState);
   state.cameraState = open ? value : "";
   els.cameraDrawer.classList.toggle("is-open", open);
   els.conversation.classList.toggle("has-camera", open);
@@ -226,9 +254,11 @@ function syncCameraDrawer(value) {
   if (open) {
     els.cameraState.textContent = `State ${value}`;
     els.cameraQuestion.textContent = prompt;
+    renderSampleThumbnails();
   } else {
     els.cameraState.textContent = "State -";
     els.cameraQuestion.textContent = "";
+    if (wasOpen) resetCameraInput();
   }
   setCameraButtonEnabled();
 }
@@ -260,6 +290,35 @@ function addBubble(role, text) {
   return bubble;
 }
 
+// Render a single chat bubble holding an image and (optionally) text together.
+function addImageBubble(role, imageUrl, text) {
+  if (els.chatLog.querySelector(".chat__empty")) {
+    els.chatLog.innerHTML = "";
+  }
+  const bubble = document.createElement("div");
+  bubble.className = `bubble bubble--${role}`;
+  if (role !== "system") {
+    const tag = document.createElement("span");
+    tag.className = "bubble__role";
+    tag.textContent = role === "user" ? "You" : "Assistant";
+    bubble.appendChild(tag);
+  }
+  const img = document.createElement("img");
+  img.className = "bubble__image";
+  img.src = imageUrl;
+  img.alt = text || "image";
+  bubble.appendChild(img);
+
+  const body = document.createElement("span");
+  body.className = "bubble__text";
+  body.textContent = text || "";
+  bubble.appendChild(body);
+
+  els.chatLog.appendChild(bubble);
+  scrollChatToBottom();
+  return bubble;
+}
+
 function appendToBubble(bubble, text) {
   const body = bubble.querySelector(".bubble__text");
   body.textContent += text;
@@ -499,6 +558,9 @@ function compactWsPayload(payload) {
   if (typeof compact.audio === "string") {
     compact.audio = `<base64 ${compact.audio.length} chars>`;
   }
+  if (typeof compact.image === "string") {
+    compact.image = `<base64 ${compact.image.length} chars>`;
+  }
   if (typeof compact.data === "string" && compact.data.length > 160) {
     compact.data = `<string ${compact.data.length} chars>`;
   }
@@ -807,6 +869,219 @@ function resetPlaybackClock() {
   }
 }
 
+/* ------------------------------------------------------ Camera / image */
+
+function setPreviewMode(mode) {
+  // mode: "camera" | "photo" | "idle"
+  els.cameraPreview.classList.toggle("is-camera", mode === "camera");
+  els.cameraPreview.classList.toggle("is-photo", mode === "photo");
+}
+
+// Draw an <img>/<video> source to the canvas and return a normalized payload
+// (JPEG data URL + dimensions) suitable for an `input.image` message.
+function mediaToPayload(source) {
+  const srcW = source.videoWidth || source.naturalWidth || source.width;
+  const srcH = source.videoHeight || source.naturalHeight || source.height;
+  if (!srcW || !srcH) return null;
+
+  let w = srcW;
+  let h = srcH;
+  const longest = Math.max(w, h);
+  if (longest > IMAGE_MAX_DIM) {
+    const scale = IMAGE_MAX_DIM / longest;
+    w = Math.round(w * scale);
+    h = Math.round(h * scale);
+  }
+
+  const canvas = els.cameraCanvas;
+  canvas.width = w;
+  canvas.height = h;
+  const ctx = canvas.getContext("2d");
+  ctx.drawImage(source, 0, 0, w, h);
+
+  let dataUrl;
+  try {
+    dataUrl = canvas.toDataURL("image/jpeg", IMAGE_JPEG_QUALITY);
+  } catch (err) {
+    addWsLog("system", `image encode failed: ${err.message || err}`);
+    return null;
+  }
+  return { dataUrl, mime: "image/jpeg", width: w, height: h };
+}
+
+function setPendingImage(payload) {
+  state.pendingImage = payload;
+  if (payload) {
+    els.cameraPhoto.src = payload.dataUrl;
+    setPreviewMode("photo");
+  }
+  setCameraButtonEnabled();
+}
+
+async function startCamera() {
+  if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
+    addWsLog("system", "getUserMedia not available in this browser");
+    return;
+  }
+  stopCameraStream();
+  try {
+    state.cameraStream = await navigator.mediaDevices.getUserMedia({
+      video: { facingMode: state.cameraFacing },
+      audio: false,
+    });
+  } catch (err) {
+    addWsLog("system", `camera error: ${err.message || err}`);
+    return;
+  }
+  els.cameraVideo.srcObject = state.cameraStream;
+  try {
+    await els.cameraVideo.play();
+  } catch (_) {
+    /* autoplay may resolve later */
+  }
+  state.cameraActive = true;
+  state.pendingImage = null;
+  setPreviewMode("camera");
+  els.cameraStartBtn.classList.add("is-active");
+  els.cameraStartBtn.textContent = "重新拍摄";
+  els.cameraFlipBtn.hidden = false;
+  clearSampleSelection();
+  setCameraButtonEnabled();
+}
+
+function stopCameraStream() {
+  if (state.cameraStream) {
+    state.cameraStream.getTracks().forEach((track) => track.stop());
+    state.cameraStream = null;
+  }
+  els.cameraVideo.srcObject = null;
+  state.cameraActive = false;
+  els.cameraStartBtn.classList.remove("is-active");
+  els.cameraStartBtn.textContent = "使用摄像头";
+  els.cameraFlipBtn.hidden = true;
+}
+
+function captureFromCamera() {
+  const payload = mediaToPayload(els.cameraVideo);
+  if (!payload) return null;
+  stopCameraStream();
+  setPendingImage(payload);
+  return payload;
+}
+
+// Load a same-origin/object URL into an <img> and resolve once decoded.
+function loadImage(src) {
+  return new Promise((resolve, reject) => {
+    const img = new Image();
+    img.onload = () => resolve(img);
+    img.onerror = () => reject(new Error(`failed to load image: ${src}`));
+    img.src = src;
+  });
+}
+
+async function selectFileImage(file) {
+  if (!file) return;
+  const objectUrl = URL.createObjectURL(file);
+  try {
+    const img = await loadImage(objectUrl);
+    const payload = mediaToPayload(img);
+    if (!payload) return;
+    stopCameraStream();
+    clearSampleSelection();
+    setPendingImage(payload);
+  } catch (err) {
+    addWsLog("system", `upload error: ${err.message || err}`);
+  } finally {
+    URL.revokeObjectURL(objectUrl);
+  }
+}
+
+async function selectSampleImage(src, buttonEl) {
+  try {
+    const img = await loadImage(src);
+    const payload = mediaToPayload(img);
+    if (!payload) return;
+    stopCameraStream();
+    clearSampleSelection();
+    if (buttonEl) buttonEl.classList.add("is-selected");
+    setPendingImage(payload);
+  } catch (err) {
+    addWsLog("system", `sample error: ${err.message || err}`);
+  }
+}
+
+function clearSampleSelection() {
+  els.cameraSamples
+    .querySelectorAll(".camera-drawer__sample.is-selected")
+    .forEach((el) => el.classList.remove("is-selected"));
+}
+
+function renderSampleThumbnails() {
+  if (state.samplesRendered) return;
+  state.samplesRendered = true;
+  els.cameraSamples.innerHTML = "";
+  for (const sample of SAMPLE_IMAGES) {
+    const btn = document.createElement("button");
+    btn.type = "button";
+    btn.className = "camera-drawer__sample";
+    btn.title = sample.label;
+    const img = document.createElement("img");
+    img.src = sample.src;
+    img.alt = sample.label;
+    btn.appendChild(img);
+    btn.addEventListener("click", () => selectSampleImage(sample.src, btn));
+    els.cameraSamples.appendChild(btn);
+  }
+}
+
+function resetCameraInput() {
+  stopCameraStream();
+  state.pendingImage = null;
+  clearSampleSelection();
+  els.cameraPhoto.removeAttribute("src");
+  setPreviewMode("idle");
+  setCameraButtonEnabled();
+}
+
+function sendImage(payload, text) {
+  if (!payload) return false;
+  if (!state.ws || state.ws.readyState !== WebSocket.OPEN) return false;
+
+  const message = {
+    type: "input.image",
+    image: payload.dataUrl,
+    mime_type: payload.mime,
+    width: payload.width,
+    height: payload.height,
+    text: text || CAMERA_DONE_TEXT,
+    interrupt: true,
+  };
+
+  wsSend(JSON.stringify(message));
+  // Mirror the text-input path: interrupt in-flight bot audio and render the
+  // user's image + text together as one local bubble (the engine does not echo
+  // image input back as a transcript event).
+  stopPlaybackQueue();
+  state.currentAssistantBubble = null;
+  addImageBubble("user", payload.dataUrl, text || CAMERA_DONE_TEXT);
+  return true;
+}
+
+function submitCameraImage() {
+  // If the live camera is on, grab the current frame first; otherwise use the
+  // already-selected (uploaded / sample / captured) image.
+  let payload = state.pendingImage;
+  if (state.cameraActive) {
+    payload = captureFromCamera() || payload;
+  }
+  if (!payload) return;
+  // Keep the existing workflow contract: the accompanying text stays the
+  // "【拍摄完成】" marker that advances the FastGPT camera step; the image is
+  // the new multimodal attachment.
+  if (!sendImage(payload, CAMERA_DONE_TEXT)) return;
+  resetCameraInput();
+}
+
 /* --------------------------------------------------------- Chat updates */
 
 function handleUserTranscript(text) {
@@ -1139,7 +1414,23 @@ els.clearWsLogBtn.addEventListener("click", () => {
 
 els.cameraDoneBtn.addEventListener("click", () => {
   if (!state.cameraState) return;
-  sendText(CAMERA_DONE_TEXT);
+  submitCameraImage();
+});
+
+els.cameraStartBtn.addEventListener("click", () => {
+  startCamera();
+});
+
+els.cameraFlipBtn.addEventListener("click", () => {
+  state.cameraFacing =
+    state.cameraFacing === "environment" ? "user" : "environment";
+  if (state.cameraActive) startCamera();
+});
+
+els.cameraUpload.addEventListener("change", (event) => {
+  const file = event.target.files && event.target.files[0];
+  selectFileImage(file);
+  event.target.value = "";
 });
 
 function autosizeTextarea() {
@@ -1174,6 +1465,7 @@ els.textInput.addEventListener("keydown", (event) => {
 });
 
 window.addEventListener("beforeunload", () => {
+  stopCameraStream();
   if (state.ws) {
     try {
       state.ws.close();

examples/webpage/index.html

@@ -82,16 +82,65 @@
                   <span id="camera-state" class="camera-drawer__state">State -</span>
                 </div>
 
-                <div class="camera-drawer__preview" aria-hidden="true">
+                <div id="camera-preview" class="camera-drawer__preview">
+                  <video
+                    id="camera-video"
+                    class="camera-drawer__video"
+                    playsinline
+                    muted
+                    autoplay
+                  ></video>
+                  <img
+                    id="camera-photo"
+                    class="camera-drawer__photo"
+                    alt="Selected image preview"
+                  />
                   <span class="camera-drawer__corner camera-drawer__corner--tl"></span>
                   <span class="camera-drawer__corner camera-drawer__corner--tr"></span>
                   <span class="camera-drawer__corner camera-drawer__corner--bl"></span>
                   <span class="camera-drawer__corner camera-drawer__corner--br"></span>
                   <span class="camera-drawer__lens"></span>
                   <span class="camera-drawer__scan"></span>
+                  <span id="camera-placeholder" class="camera-drawer__placeholder">
+                    打开摄像头实时拍摄，或从下方选择 / 上传图片
+                  </span>
                 </div>
 
                 <p id="camera-question" class="camera-drawer__question"></p>
+
+                <div class="camera-drawer__sources">
+                  <button
+                    id="camera-start-btn"
+                    class="btn btn--ghost camera-drawer__source"
+                    type="button"
+                  >
+                    使用摄像头
+                  </button>
+                  <button
+                    id="camera-flip-btn"
+                    class="btn btn--ghost camera-drawer__source"
+                    type="button"
+                    hidden
+                  >
+                    切换摄像头
+                  </button>
+                  <label class="btn btn--ghost camera-drawer__source">
+                    上传图片
+                    <input
+                      id="camera-upload"
+                      type="file"
+                      accept="image/*"
+                      hidden
+                    />
+                  </label>
+                </div>
+
+                <div
+                  id="camera-samples"
+                  class="camera-drawer__samples"
+                  aria-label="示例图片，点击选择"
+                ></div>
+
                 <button
                   id="camera-done-btn"
                   class="btn btn--primary camera-drawer__button"
@@ -100,6 +149,7 @@
                 >
                   拍摄完成
                 </button>
+                <canvas id="camera-canvas" hidden></canvas>
               </div>
             </aside>
 

examples/webpage/styles.css

@@ -148,6 +148,45 @@ body {
   background-size: 34px 34px, 34px 34px, auto, auto;
 }
 
+.camera-drawer__video,
+.camera-drawer__photo {
+  position: absolute;
+  inset: 0;
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+  display: none;
+  z-index: 1;
+}
+
+.camera-drawer__preview.is-camera .camera-drawer__video {
+  display: block;
+}
+
+.camera-drawer__preview.is-photo .camera-drawer__photo {
+  display: block;
+}
+
+/* Hide the decorative lens/scan/placeholder once real media is showing. */
+.camera-drawer__preview.is-camera .camera-drawer__lens,
+.camera-drawer__preview.is-photo .camera-drawer__lens,
+.camera-drawer__preview.is-camera .camera-drawer__scan,
+.camera-drawer__preview.is-photo .camera-drawer__scan,
+.camera-drawer__preview.is-camera .camera-drawer__placeholder,
+.camera-drawer__preview.is-photo .camera-drawer__placeholder {
+  display: none;
+}
+
+.camera-drawer__placeholder {
+  position: absolute;
+  inset: auto 18px 16px;
+  z-index: 2;
+  color: rgba(214, 220, 235, 0.78);
+  font-size: 12px;
+  line-height: 1.5;
+  text-align: center;
+}
+
 .camera-drawer__lens {
   position: absolute;
   top: 50%;
@@ -174,6 +213,7 @@ body {
 
 .camera-drawer__corner {
   position: absolute;
+  z-index: 2;
   width: 28px;
   height: 28px;
   border-color: rgba(255, 255, 255, 0.7);
@@ -229,6 +269,62 @@ body {
   cursor: not-allowed;
 }
 
+.camera-drawer__sources {
+  display: flex;
+  flex-wrap: wrap;
+  gap: 8px;
+}
+
+.camera-drawer__source {
+  flex: 1 1 auto;
+  min-height: 38px;
+  font-size: 13px;
+  font-weight: 600;
+  cursor: pointer;
+}
+
+.camera-drawer__source.is-active {
+  border-color: var(--success);
+  color: var(--success);
+}
+
+.camera-drawer__samples {
+  display: grid;
+  grid-template-columns: repeat(4, 1fr);
+  gap: 8px;
+}
+
+.camera-drawer__samples:empty {
+  display: none;
+}
+
+.camera-drawer__sample {
+  position: relative;
+  aspect-ratio: 4 / 3;
+  padding: 0;
+  border: 2px solid transparent;
+  border-radius: 10px;
+  overflow: hidden;
+  cursor: pointer;
+  background: #0f141f;
+}
+
+.camera-drawer__sample img {
+  width: 100%;
+  height: 100%;
+  object-fit: cover;
+  display: block;
+}
+
+.camera-drawer__sample:hover {
+  border-color: rgba(149, 160, 187, 0.6);
+}
+
+.camera-drawer__sample.is-selected {
+  border-color: var(--success);
+  box-shadow: 0 0 0 1px var(--success);
+}
+
 .app__body {
   display: grid;
   grid-template-columns: minmax(0, 1fr) clamp(300px, 32vw, 420px);
@@ -511,6 +607,18 @@ body {
   margin-bottom: 4px;
 }
 
+.bubble__image {
+  display: block;
+  max-width: 240px;
+  width: 100%;
+  border-radius: 10px;
+  margin-bottom: 6px;
+}
+
+.bubble__image + .bubble__text:empty {
+  display: none;
+}
+
 /* WebSocket log --------------------------------------------------------- */
 
 .ws-log {