Add image upload in conversation
This commit is contained in:
Xin Wang
@@ -24,6 +24,17 @@ const WS_LOG_GROUP_KEYS = {
|
||||
AUDIO_SEND: "send:input.audio",
|
||||
};
|
||||
const CAMERA_DONE_TEXT = "【拍摄完成】";
|
||||
// Sample images shown as thumbnails under the camera preview. Same-origin files
|
||||
// so they can be drawn to a canvas (for base64 + dimensions) without tainting.
|
||||
const SAMPLE_IMAGES = [
|
||||
{ src: "./samples/front-damage.jpg", label: "车辆前部" },
|
||||
{ src: "./samples/plate.jpg", label: "车牌" },
|
||||
{ src: "./samples/license.jpg", label: "驾驶证" },
|
||||
{ src: "./samples/scene.jpg", label: "事故现场" },
|
||||
];
|
||||
// Cap the longer edge before JPEG-encoding so payloads stay small.
|
||||
const IMAGE_MAX_DIM = 1280;
|
||||
const IMAGE_JPEG_QUALITY = 0.85;
|
||||
const CAMERA_STATE_PROMPTS = {
|
||||
2000: "请对准车辆碰撞部位拍摄照片。",
|
||||
2001: "请对准车辆碰撞部位拍摄照片。",
|
||||
@@ -62,6 +73,14 @@ const els = {
|
||||
cameraState: document.getElementById("camera-state"),
|
||||
cameraQuestion: document.getElementById("camera-question"),
|
||||
cameraDoneBtn: document.getElementById("camera-done-btn"),
|
||||
cameraPreview: document.getElementById("camera-preview"),
|
||||
cameraVideo: document.getElementById("camera-video"),
|
||||
cameraPhoto: document.getElementById("camera-photo"),
|
||||
cameraCanvas: document.getElementById("camera-canvas"),
|
||||
cameraStartBtn: document.getElementById("camera-start-btn"),
|
||||
cameraFlipBtn: document.getElementById("camera-flip-btn"),
|
||||
cameraUpload: document.getElementById("camera-upload"),
|
||||
cameraSamples: document.getElementById("camera-samples"),
|
||||
clearBtn: document.getElementById("clear-btn"),
|
||||
clearWsLogBtn: document.getElementById("clear-ws-log-btn"),
|
||||
wsLog: document.getElementById("ws-log"),
|
||||
@@ -125,6 +144,13 @@ const state = {
|
||||
assistantState: "",
|
||||
cameraState: "",
|
||||
|
||||
// Camera / image input.
|
||||
cameraStream: null,
|
||||
cameraActive: false,
|
||||
cameraFacing: "environment",
|
||||
pendingImage: null,
|
||||
samplesRendered: false,
|
||||
|
||||
// VU meter smoothing.
|
||||
meterLevel: 0,
|
||||
|
||||
@@ -211,14 +237,16 @@ function setAssistantState(value) {
|
||||
|
||||
function setCameraButtonEnabled() {
|
||||
if (!els.cameraDoneBtn) return;
|
||||
els.cameraDoneBtn.disabled =
|
||||
!state.connected || !state.cameraState ||
|
||||
!state.ws || state.ws.readyState !== WebSocket.OPEN;
|
||||
const wsReady =
|
||||
state.connected && state.ws && state.ws.readyState === WebSocket.OPEN;
|
||||
const hasImageSource = state.cameraActive || Boolean(state.pendingImage);
|
||||
els.cameraDoneBtn.disabled = !wsReady || !state.cameraState || !hasImageSource;
|
||||
}
|
||||
|
||||
function syncCameraDrawer(value) {
|
||||
const prompt = CAMERA_STATE_PROMPTS[value];
|
||||
const open = Boolean(prompt);
|
||||
const wasOpen = Boolean(state.cameraState);
|
||||
state.cameraState = open ? value : "";
|
||||
els.cameraDrawer.classList.toggle("is-open", open);
|
||||
els.conversation.classList.toggle("has-camera", open);
|
||||
@@ -226,9 +254,11 @@ function syncCameraDrawer(value) {
|
||||
if (open) {
|
||||
els.cameraState.textContent = `State ${value}`;
|
||||
els.cameraQuestion.textContent = prompt;
|
||||
renderSampleThumbnails();
|
||||
} else {
|
||||
els.cameraState.textContent = "State -";
|
||||
els.cameraQuestion.textContent = "";
|
||||
if (wasOpen) resetCameraInput();
|
||||
}
|
||||
setCameraButtonEnabled();
|
||||
}
|
||||
@@ -260,6 +290,35 @@ function addBubble(role, text) {
|
||||
return bubble;
|
||||
}
|
||||
|
||||
// Render a single chat bubble holding an image and (optionally) text together.
|
||||
function addImageBubble(role, imageUrl, text) {
|
||||
if (els.chatLog.querySelector(".chat__empty")) {
|
||||
els.chatLog.innerHTML = "";
|
||||
}
|
||||
const bubble = document.createElement("div");
|
||||
bubble.className = `bubble bubble--${role}`;
|
||||
if (role !== "system") {
|
||||
const tag = document.createElement("span");
|
||||
tag.className = "bubble__role";
|
||||
tag.textContent = role === "user" ? "You" : "Assistant";
|
||||
bubble.appendChild(tag);
|
||||
}
|
||||
const img = document.createElement("img");
|
||||
img.className = "bubble__image";
|
||||
img.src = imageUrl;
|
||||
img.alt = text || "image";
|
||||
bubble.appendChild(img);
|
||||
|
||||
const body = document.createElement("span");
|
||||
body.className = "bubble__text";
|
||||
body.textContent = text || "";
|
||||
bubble.appendChild(body);
|
||||
|
||||
els.chatLog.appendChild(bubble);
|
||||
scrollChatToBottom();
|
||||
return bubble;
|
||||
}
|
||||
|
||||
function appendToBubble(bubble, text) {
|
||||
const body = bubble.querySelector(".bubble__text");
|
||||
body.textContent += text;
|
||||
@@ -499,6 +558,9 @@ function compactWsPayload(payload) {
|
||||
if (typeof compact.audio === "string") {
|
||||
compact.audio = `<base64 ${compact.audio.length} chars>`;
|
||||
}
|
||||
if (typeof compact.image === "string") {
|
||||
compact.image = `<base64 ${compact.image.length} chars>`;
|
||||
}
|
||||
if (typeof compact.data === "string" && compact.data.length > 160) {
|
||||
compact.data = `<string ${compact.data.length} chars>`;
|
||||
}
|
||||
@@ -807,6 +869,219 @@ function resetPlaybackClock() {
|
||||
}
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------ Camera / image */
|
||||
|
||||
function setPreviewMode(mode) {
|
||||
// mode: "camera" | "photo" | "idle"
|
||||
els.cameraPreview.classList.toggle("is-camera", mode === "camera");
|
||||
els.cameraPreview.classList.toggle("is-photo", mode === "photo");
|
||||
}
|
||||
|
||||
// Draw an <img>/<video> source to the canvas and return a normalized payload
|
||||
// (JPEG data URL + dimensions) suitable for an `input.image` message.
|
||||
function mediaToPayload(source) {
|
||||
const srcW = source.videoWidth || source.naturalWidth || source.width;
|
||||
const srcH = source.videoHeight || source.naturalHeight || source.height;
|
||||
if (!srcW || !srcH) return null;
|
||||
|
||||
let w = srcW;
|
||||
let h = srcH;
|
||||
const longest = Math.max(w, h);
|
||||
if (longest > IMAGE_MAX_DIM) {
|
||||
const scale = IMAGE_MAX_DIM / longest;
|
||||
w = Math.round(w * scale);
|
||||
h = Math.round(h * scale);
|
||||
}
|
||||
|
||||
const canvas = els.cameraCanvas;
|
||||
canvas.width = w;
|
||||
canvas.height = h;
|
||||
const ctx = canvas.getContext("2d");
|
||||
ctx.drawImage(source, 0, 0, w, h);
|
||||
|
||||
let dataUrl;
|
||||
try {
|
||||
dataUrl = canvas.toDataURL("image/jpeg", IMAGE_JPEG_QUALITY);
|
||||
} catch (err) {
|
||||
addWsLog("system", `image encode failed: ${err.message || err}`);
|
||||
return null;
|
||||
}
|
||||
return { dataUrl, mime: "image/jpeg", width: w, height: h };
|
||||
}
|
||||
|
||||
function setPendingImage(payload) {
|
||||
state.pendingImage = payload;
|
||||
if (payload) {
|
||||
els.cameraPhoto.src = payload.dataUrl;
|
||||
setPreviewMode("photo");
|
||||
}
|
||||
setCameraButtonEnabled();
|
||||
}
|
||||
|
||||
async function startCamera() {
|
||||
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
|
||||
addWsLog("system", "getUserMedia not available in this browser");
|
||||
return;
|
||||
}
|
||||
stopCameraStream();
|
||||
try {
|
||||
state.cameraStream = await navigator.mediaDevices.getUserMedia({
|
||||
video: { facingMode: state.cameraFacing },
|
||||
audio: false,
|
||||
});
|
||||
} catch (err) {
|
||||
addWsLog("system", `camera error: ${err.message || err}`);
|
||||
return;
|
||||
}
|
||||
els.cameraVideo.srcObject = state.cameraStream;
|
||||
try {
|
||||
await els.cameraVideo.play();
|
||||
} catch (_) {
|
||||
/* autoplay may resolve later */
|
||||
}
|
||||
state.cameraActive = true;
|
||||
state.pendingImage = null;
|
||||
setPreviewMode("camera");
|
||||
els.cameraStartBtn.classList.add("is-active");
|
||||
els.cameraStartBtn.textContent = "重新拍摄";
|
||||
els.cameraFlipBtn.hidden = false;
|
||||
clearSampleSelection();
|
||||
setCameraButtonEnabled();
|
||||
}
|
||||
|
||||
function stopCameraStream() {
|
||||
if (state.cameraStream) {
|
||||
state.cameraStream.getTracks().forEach((track) => track.stop());
|
||||
state.cameraStream = null;
|
||||
}
|
||||
els.cameraVideo.srcObject = null;
|
||||
state.cameraActive = false;
|
||||
els.cameraStartBtn.classList.remove("is-active");
|
||||
els.cameraStartBtn.textContent = "使用摄像头";
|
||||
els.cameraFlipBtn.hidden = true;
|
||||
}
|
||||
|
||||
function captureFromCamera() {
|
||||
const payload = mediaToPayload(els.cameraVideo);
|
||||
if (!payload) return null;
|
||||
stopCameraStream();
|
||||
setPendingImage(payload);
|
||||
return payload;
|
||||
}
|
||||
|
||||
// Load a same-origin/object URL into an <img> and resolve once decoded.
|
||||
function loadImage(src) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const img = new Image();
|
||||
img.onload = () => resolve(img);
|
||||
img.onerror = () => reject(new Error(`failed to load image: ${src}`));
|
||||
img.src = src;
|
||||
});
|
||||
}
|
||||
|
||||
async function selectFileImage(file) {
|
||||
if (!file) return;
|
||||
const objectUrl = URL.createObjectURL(file);
|
||||
try {
|
||||
const img = await loadImage(objectUrl);
|
||||
const payload = mediaToPayload(img);
|
||||
if (!payload) return;
|
||||
stopCameraStream();
|
||||
clearSampleSelection();
|
||||
setPendingImage(payload);
|
||||
} catch (err) {
|
||||
addWsLog("system", `upload error: ${err.message || err}`);
|
||||
} finally {
|
||||
URL.revokeObjectURL(objectUrl);
|
||||
}
|
||||
}
|
||||
|
||||
async function selectSampleImage(src, buttonEl) {
|
||||
try {
|
||||
const img = await loadImage(src);
|
||||
const payload = mediaToPayload(img);
|
||||
if (!payload) return;
|
||||
stopCameraStream();
|
||||
clearSampleSelection();
|
||||
if (buttonEl) buttonEl.classList.add("is-selected");
|
||||
setPendingImage(payload);
|
||||
} catch (err) {
|
||||
addWsLog("system", `sample error: ${err.message || err}`);
|
||||
}
|
||||
}
|
||||
|
||||
function clearSampleSelection() {
|
||||
els.cameraSamples
|
||||
.querySelectorAll(".camera-drawer__sample.is-selected")
|
||||
.forEach((el) => el.classList.remove("is-selected"));
|
||||
}
|
||||
|
||||
function renderSampleThumbnails() {
|
||||
if (state.samplesRendered) return;
|
||||
state.samplesRendered = true;
|
||||
els.cameraSamples.innerHTML = "";
|
||||
for (const sample of SAMPLE_IMAGES) {
|
||||
const btn = document.createElement("button");
|
||||
btn.type = "button";
|
||||
btn.className = "camera-drawer__sample";
|
||||
btn.title = sample.label;
|
||||
const img = document.createElement("img");
|
||||
img.src = sample.src;
|
||||
img.alt = sample.label;
|
||||
btn.appendChild(img);
|
||||
btn.addEventListener("click", () => selectSampleImage(sample.src, btn));
|
||||
els.cameraSamples.appendChild(btn);
|
||||
}
|
||||
}
|
||||
|
||||
function resetCameraInput() {
|
||||
stopCameraStream();
|
||||
state.pendingImage = null;
|
||||
clearSampleSelection();
|
||||
els.cameraPhoto.removeAttribute("src");
|
||||
setPreviewMode("idle");
|
||||
setCameraButtonEnabled();
|
||||
}
|
||||
|
||||
function sendImage(payload, text) {
|
||||
if (!payload) return false;
|
||||
if (!state.ws || state.ws.readyState !== WebSocket.OPEN) return false;
|
||||
|
||||
const message = {
|
||||
type: "input.image",
|
||||
image: payload.dataUrl,
|
||||
mime_type: payload.mime,
|
||||
width: payload.width,
|
||||
height: payload.height,
|
||||
text: text || CAMERA_DONE_TEXT,
|
||||
interrupt: true,
|
||||
};
|
||||
|
||||
wsSend(JSON.stringify(message));
|
||||
// Mirror the text-input path: interrupt in-flight bot audio and render the
|
||||
// user's image + text together as one local bubble (the engine does not echo
|
||||
// image input back as a transcript event).
|
||||
stopPlaybackQueue();
|
||||
state.currentAssistantBubble = null;
|
||||
addImageBubble("user", payload.dataUrl, text || CAMERA_DONE_TEXT);
|
||||
return true;
|
||||
}
|
||||
|
||||
function submitCameraImage() {
|
||||
// If the live camera is on, grab the current frame first; otherwise use the
|
||||
// already-selected (uploaded / sample / captured) image.
|
||||
let payload = state.pendingImage;
|
||||
if (state.cameraActive) {
|
||||
payload = captureFromCamera() || payload;
|
||||
}
|
||||
if (!payload) return;
|
||||
// Keep the existing workflow contract: the accompanying text stays the
|
||||
// "【拍摄完成】" marker that advances the FastGPT camera step; the image is
|
||||
// the new multimodal attachment.
|
||||
if (!sendImage(payload, CAMERA_DONE_TEXT)) return;
|
||||
resetCameraInput();
|
||||
}
|
||||
|
||||
/* --------------------------------------------------------- Chat updates */
|
||||
|
||||
function handleUserTranscript(text) {
|
||||
@@ -1139,7 +1414,23 @@ els.clearWsLogBtn.addEventListener("click", () => {
|
||||
|
||||
els.cameraDoneBtn.addEventListener("click", () => {
|
||||
if (!state.cameraState) return;
|
||||
sendText(CAMERA_DONE_TEXT);
|
||||
submitCameraImage();
|
||||
});
|
||||
|
||||
els.cameraStartBtn.addEventListener("click", () => {
|
||||
startCamera();
|
||||
});
|
||||
|
||||
els.cameraFlipBtn.addEventListener("click", () => {
|
||||
state.cameraFacing =
|
||||
state.cameraFacing === "environment" ? "user" : "environment";
|
||||
if (state.cameraActive) startCamera();
|
||||
});
|
||||
|
||||
els.cameraUpload.addEventListener("change", (event) => {
|
||||
const file = event.target.files && event.target.files[0];
|
||||
selectFileImage(file);
|
||||
event.target.value = "";
|
||||
});
|
||||
|
||||
function autosizeTextarea() {
|
||||
@@ -1174,6 +1465,7 @@ els.textInput.addEventListener("keydown", (event) => {
|
||||
});
|
||||
|
||||
window.addEventListener("beforeunload", () => {
|
||||
stopCameraStream();
|
||||
if (state.ws) {
|
||||
try {
|
||||
state.ws.close();
|
||||
|
||||
Reference in New Issue
Block a user