Sync with engine v5

This commit is contained in:
Xin Wang
2026-06-03 12:36:18 +08:00
parent 056a8a4ad8
commit 705a63dd25
17 changed files with 854 additions and 111 deletions

View File

@@ -24,6 +24,19 @@ const WS_LOG_GROUP_KEYS = {
AUDIO_SEND: "send:input.audio",
};
const CAMERA_DONE_TEXT = "【拍摄完成】";
// Sample images shown as thumbnails under the camera preview. Same-origin files
// so they can be drawn to a canvas (for base64 + dimensions) without tainting.
const SAMPLE_IMAGES = [
{ src: "./samples/damage1.png", label: "车辆前部" },
{ src: "./samples/damage2.png", label: "车辆后部" },
{ src: "./samples/plate1.jpg", label: "车牌 1" },
{ src: "./samples/plate2.jpg", label: "车牌 2" },
{ src: "./samples/user1.jpg", label: "人物 1" },
{ src: "./samples/user2.jpg", label: "人物 2" },
];
// Cap the longer edge before JPEG-encoding so payloads stay small.
const IMAGE_MAX_DIM = 1280;
const IMAGE_JPEG_QUALITY = 0.85;
const CAMERA_STATE_PROMPTS = {
2000: "请对准车辆碰撞部位拍摄照片。",
2001: "请对准车辆碰撞部位拍摄照片。",
@@ -62,6 +75,15 @@ const els = {
cameraState: document.getElementById("camera-state"),
cameraQuestion: document.getElementById("camera-question"),
cameraDoneBtn: document.getElementById("camera-done-btn"),
cameraPreview: document.getElementById("camera-preview"),
cameraVideo: document.getElementById("camera-video"),
cameraPhoto: document.getElementById("camera-photo"),
cameraCanvas: document.getElementById("camera-canvas"),
cameraStartBtn: document.getElementById("camera-start-btn"),
cameraDeviceRow: document.getElementById("camera-device-row"),
cameraDeviceSelect: document.getElementById("camera-device-select"),
cameraUpload: document.getElementById("camera-upload"),
cameraSamples: document.getElementById("camera-samples"),
clearBtn: document.getElementById("clear-btn"),
clearWsLogBtn: document.getElementById("clear-ws-log-btn"),
wsLog: document.getElementById("ws-log"),
@@ -125,6 +147,14 @@ const state = {
assistantState: "",
cameraState: "",
// Camera / image input.
cameraStream: null,
cameraActive: false,
cameraFacing: "environment",
videoDevices: [],
pendingImage: null,
samplesRendered: false,
// VU meter smoothing.
meterLevel: 0,
@@ -143,15 +173,15 @@ function setConnectButton() {
els.chatId.disabled = state.connected || state.connecting;
els.copyChatIdBtn.disabled = !state.connected || !state.chatId;
if (state.connecting) {
els.connectBtn.textContent = "Connecting…";
els.connectBtn.textContent = "连接中…";
els.connectBtn.disabled = true;
els.connectBtn.classList.remove("is-disconnect");
} else if (state.connected) {
els.connectBtn.textContent = "Disconnect";
els.connectBtn.textContent = "断开连接";
els.connectBtn.disabled = false;
els.connectBtn.classList.add("is-disconnect");
} else {
els.connectBtn.textContent = "Connect";
els.connectBtn.textContent = "连接";
els.connectBtn.disabled = false;
els.connectBtn.classList.remove("is-disconnect");
}
@@ -180,8 +210,8 @@ async function copyChatId() {
function setMicButton() {
els.micBtn.disabled = !state.connected;
els.micBtn.setAttribute("aria-pressed", state.micEnabled ? "true" : "false");
els.micBtn.title = state.micEnabled ? "Mute mic" : "Unmute mic";
els.micLabel.textContent = state.micEnabled ? "Mute mic" : "Enable mic";
els.micBtn.title = state.micEnabled ? "关闭麦克风" : "开启麦克风";
els.micLabel.textContent = state.micEnabled ? "关闭麦克风" : "开启麦克风";
els.micIndicator.classList.toggle("is-active", state.micEnabled);
}
@@ -204,41 +234,40 @@ function setAssistantState(value) {
const label = text.length > 32 ? `${text.slice(0, 31)}` : text;
state.assistantState = text;
els.stateIndicator.classList.toggle("is-active", Boolean(text));
els.stateLabel.textContent = label ? `State ${label}` : "State -";
els.stateIndicator.title = label ? `Assistant state: ${text}` : "Assistant state";
els.stateLabel.textContent = label ? `状态 ${label}` : "状态 -";
els.stateIndicator.title = label ? `助手状态:${text}` : "助手状态";
syncCameraDrawer(text);
}
function setCameraButtonEnabled() {
if (!els.cameraDoneBtn) return;
els.cameraDoneBtn.disabled =
!state.connected || !state.cameraState ||
!state.ws || state.ws.readyState !== WebSocket.OPEN;
const wsReady =
state.connected && state.ws && state.ws.readyState === WebSocket.OPEN;
const hasImageSource = state.cameraActive || Boolean(state.pendingImage);
els.cameraDoneBtn.disabled = !wsReady || !state.cameraState || !hasImageSource;
}
function syncCameraDrawer(value) {
const prompt = CAMERA_STATE_PROMPTS[value];
const open = Boolean(prompt);
const wasOpen = Boolean(state.cameraState);
state.cameraState = open ? value : "";
els.cameraDrawer.classList.toggle("is-open", open);
els.conversation.classList.toggle("has-camera", open);
els.cameraDrawer.setAttribute("aria-hidden", open ? "false" : "true");
if (open) {
els.cameraState.textContent = `State ${value}`;
els.cameraState.textContent = `状态 ${value}`;
els.cameraQuestion.textContent = prompt;
renderSampleThumbnails();
selectDefaultImage();
} else {
els.cameraState.textContent = "State -";
els.cameraState.textContent = "状态 -";
els.cameraQuestion.textContent = "";
if (wasOpen) resetCameraInput();
}
setCameraButtonEnabled();
}
function updateCameraQuestion(text) {
const value = typeof text === "string" ? text.trim() : "";
if (!state.cameraState || !value) return;
els.cameraQuestion.textContent = value;
}
function addBubble(role, text) {
if (els.chatLog.querySelector(".chat__empty")) {
els.chatLog.innerHTML = "";
@@ -248,7 +277,7 @@ function addBubble(role, text) {
if (role !== "system") {
const tag = document.createElement("span");
tag.className = "bubble__role";
tag.textContent = role === "user" ? "You" : "Assistant";
tag.textContent = role === "user" ? "" : "助手";
bubble.appendChild(tag);
}
const body = document.createElement("span");
@@ -260,6 +289,35 @@ function addBubble(role, text) {
return bubble;
}
// Render a single chat bubble holding an image and (optionally) text together.
function addImageBubble(role, imageUrl, text) {
if (els.chatLog.querySelector(".chat__empty")) {
els.chatLog.innerHTML = "";
}
const bubble = document.createElement("div");
bubble.className = `bubble bubble--${role}`;
if (role !== "system") {
const tag = document.createElement("span");
tag.className = "bubble__role";
tag.textContent = role === "user" ? "你" : "助手";
bubble.appendChild(tag);
}
const img = document.createElement("img");
img.className = "bubble__image";
img.src = imageUrl;
img.alt = text || "image";
bubble.appendChild(img);
const body = document.createElement("span");
body.className = "bubble__text";
body.textContent = text || "";
bubble.appendChild(body);
els.chatLog.appendChild(bubble);
scrollChatToBottom();
return bubble;
}
function appendToBubble(bubble, text) {
const body = bubble.querySelector(".bubble__text");
body.textContent += text;
@@ -276,7 +334,7 @@ function clearChat() {
setAssistantState("");
const empty = document.createElement("div");
empty.className = "chat__empty";
empty.innerHTML = "<p>Chat cleared.</p>";
empty.innerHTML = "<p>对话已清空。</p>";
els.chatLog.appendChild(empty);
}
@@ -499,6 +557,9 @@ function compactWsPayload(payload) {
if (typeof compact.audio === "string") {
compact.audio = `<base64 ${compact.audio.length} chars>`;
}
if (typeof compact.image === "string") {
compact.image = `<base64 ${compact.image.length} chars>`;
}
if (typeof compact.data === "string" && compact.data.length > 160) {
compact.data = `<string ${compact.data.length} chars>`;
}
@@ -595,7 +656,7 @@ function wsSend(data) {
function clearWsLog() {
state.wsLogGroup = null;
els.wsLog.innerHTML =
'<div class="ws-log__empty">No websocket events yet.</div>';
'<div class="ws-log__empty">暂无 WebSocket 事件。</div>';
}
/* ---------------------------------------------------------------- Audio */
@@ -618,13 +679,13 @@ function renderMicDevices() {
const defaultOption = document.createElement("option");
defaultOption.value = "";
defaultOption.textContent = "Default microphone";
defaultOption.textContent = "默认麦克风";
els.micSelect.appendChild(defaultOption);
state.micDevices.forEach((device, index) => {
const option = document.createElement("option");
option.value = device.deviceId;
option.textContent = device.label || `Microphone ${index + 1}`;
option.textContent = device.label || `麦克风 ${index + 1}`;
els.micSelect.appendChild(option);
});
@@ -691,7 +752,7 @@ async function startMic() {
state.micSourceNode.connect(state.recorderNode);
state.micEnabled = true;
addWsLog("system", "mic capture started (binary input.audio frames)");
addWsLog("system", "麦克风已开启PCM 音频流)");
setMicButton();
}
@@ -727,7 +788,7 @@ function stopMic() {
state.micEnabled = false;
updateMeter(0);
if (wasEnabled) {
addWsLog("system", "mic capture stopped");
addWsLog("system", "麦克风已关闭");
}
setMicButton();
}
@@ -807,6 +868,272 @@ function resetPlaybackClock() {
}
}
/* ------------------------------------------------------ Camera / image */
function setPreviewMode(mode) {
// mode: "camera" | "photo" | "idle"
els.cameraPreview.classList.toggle("is-camera", mode === "camera");
els.cameraPreview.classList.toggle("is-photo", mode === "photo");
}
// Draw an <img>/<video> source to the canvas and return a normalized payload
// (JPEG data URL + dimensions) suitable for an `input.image` message.
function mediaToPayload(source) {
const srcW = source.videoWidth || source.naturalWidth || source.width;
const srcH = source.videoHeight || source.naturalHeight || source.height;
if (!srcW || !srcH) return null;
let w = srcW;
let h = srcH;
const longest = Math.max(w, h);
if (longest > IMAGE_MAX_DIM) {
const scale = IMAGE_MAX_DIM / longest;
w = Math.round(w * scale);
h = Math.round(h * scale);
}
const canvas = els.cameraCanvas;
canvas.width = w;
canvas.height = h;
const ctx = canvas.getContext("2d");
ctx.drawImage(source, 0, 0, w, h);
let dataUrl;
try {
dataUrl = canvas.toDataURL("image/jpeg", IMAGE_JPEG_QUALITY);
} catch (err) {
addWsLog("system", `图片编码失败:${err.message || err}`);
return null;
}
return { dataUrl, mime: "image/jpeg", width: w, height: h };
}
function setPendingImage(payload) {
state.pendingImage = payload;
if (payload) {
els.cameraPhoto.src = payload.dataUrl;
setPreviewMode("photo");
}
setCameraButtonEnabled();
}
async function refreshVideoDevices() {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
state.videoDevices = devices.filter((d) => d.kind === "videoinput");
} catch (_) {
state.videoDevices = [];
}
}
// Fill the camera dropdown from the enumerated devices. Labels are only exposed
// after camera permission has been granted, so before that we show generic
// names ("摄像头 1", …) or just the default option.
function populateDeviceSelect(activeDeviceId) {
const sel = els.cameraDeviceSelect;
sel.innerHTML = "";
if (state.videoDevices.length === 0) {
const opt = document.createElement("option");
opt.value = "";
opt.textContent = "默认摄像头";
sel.appendChild(opt);
sel.disabled = true;
return;
}
state.videoDevices.forEach((device, index) => {
const opt = document.createElement("option");
opt.value = device.deviceId;
opt.textContent = device.label || `摄像头 ${index + 1}`;
sel.appendChild(opt);
});
sel.disabled = false;
if (activeDeviceId) sel.value = activeDeviceId;
}
async function startCamera(deviceId) {
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia) {
addWsLog("system", "该浏览器不支持摄像头访问");
return;
}
stopCameraStream();
const video = deviceId
? { deviceId: { exact: deviceId } }
: { facingMode: state.cameraFacing };
try {
state.cameraStream = await navigator.mediaDevices.getUserMedia({
video,
audio: false,
});
} catch (err) {
addWsLog("system", `摄像头错误:${err.message || err}`);
return;
}
els.cameraVideo.srcObject = state.cameraStream;
try {
await els.cameraVideo.play();
} catch (_) {
/* autoplay may resolve later */
}
state.cameraActive = true;
state.pendingImage = null;
setPreviewMode("camera");
els.cameraStartBtn.classList.add("is-active");
clearSampleSelection();
// Device labels become available only after permission is granted; refresh
// the dropdown now and select whichever camera is actually streaming.
await refreshVideoDevices();
const activeId =
state.cameraStream.getVideoTracks?.()[0]?.getSettings?.().deviceId ||
deviceId;
populateDeviceSelect(activeId);
// Reveal the camera device dropdown only while the camera is in use.
els.cameraDeviceRow.hidden = false;
setCameraButtonEnabled();
}
function stopCameraStream() {
if (state.cameraStream) {
state.cameraStream.getTracks().forEach((track) => track.stop());
state.cameraStream = null;
}
els.cameraVideo.srcObject = null;
state.cameraActive = false;
els.cameraStartBtn.classList.remove("is-active");
els.cameraDeviceRow.hidden = true;
}
function captureFromCamera() {
const payload = mediaToPayload(els.cameraVideo);
if (!payload) return null;
stopCameraStream();
setPendingImage(payload);
return payload;
}
// Load a same-origin/object URL into an <img> and resolve once decoded.
function loadImage(src) {
return new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = () => reject(new Error(`failed to load image: ${src}`));
img.src = src;
});
}
async function selectFileImage(file) {
if (!file) return;
const objectUrl = URL.createObjectURL(file);
try {
const img = await loadImage(objectUrl);
const payload = mediaToPayload(img);
if (!payload) return;
stopCameraStream();
clearSampleSelection();
setPendingImage(payload);
} catch (err) {
addWsLog("system", `上传错误:${err.message || err}`);
} finally {
URL.revokeObjectURL(objectUrl);
}
}
async function selectSampleImage(src, buttonEl) {
try {
const img = await loadImage(src);
const payload = mediaToPayload(img);
if (!payload) return;
stopCameraStream();
clearSampleSelection();
if (buttonEl) buttonEl.classList.add("is-selected");
setPendingImage(payload);
} catch (err) {
addWsLog("system", `示例图加载错误:${err.message || err}`);
}
}
function clearSampleSelection() {
els.cameraSamples
.querySelectorAll(".camera-drawer__sample.is-selected")
.forEach((el) => el.classList.remove("is-selected"));
}
function renderSampleThumbnails() {
if (state.samplesRendered) return;
state.samplesRendered = true;
els.cameraSamples.innerHTML = "";
for (const sample of SAMPLE_IMAGES) {
const btn = document.createElement("button");
btn.type = "button";
btn.className = "camera-drawer__sample";
btn.title = sample.label;
const img = document.createElement("img");
img.src = sample.src;
img.alt = sample.label;
btn.appendChild(img);
btn.addEventListener("click", () => selectSampleImage(sample.src, btn));
els.cameraSamples.appendChild(btn);
}
}
function resetCameraInput() {
stopCameraStream();
state.pendingImage = null;
clearSampleSelection();
els.cameraPhoto.removeAttribute("src");
setPreviewMode("idle");
setCameraButtonEnabled();
}
// Pre-select the first sample image so "拍摄完成" is immediately pressable when
// the drawer opens, without requiring the user to capture or pick first.
function selectDefaultImage() {
if (state.pendingImage || state.cameraActive) return;
const first = els.cameraSamples.querySelector(".camera-drawer__sample");
if (first && SAMPLE_IMAGES[0]) {
selectSampleImage(SAMPLE_IMAGES[0].src, first);
}
}
function sendImage(payload, text) {
if (!payload) return false;
if (!state.ws || state.ws.readyState !== WebSocket.OPEN) return false;
const message = {
type: "input.image",
image: payload.dataUrl,
mime_type: payload.mime,
width: payload.width,
height: payload.height,
text: text || CAMERA_DONE_TEXT,
interrupt: true,
};
wsSend(JSON.stringify(message));
// Mirror the text-input path: interrupt in-flight bot audio and render the
// user's image + text together as one local bubble (the engine does not echo
// image input back as a transcript event).
stopPlaybackQueue();
state.currentAssistantBubble = null;
addImageBubble("user", payload.dataUrl, text || CAMERA_DONE_TEXT);
return true;
}
function submitCameraImage() {
// If the live camera is on, grab the current frame first; otherwise use the
// already-selected (uploaded / sample / captured) image.
let payload = state.pendingImage;
if (state.cameraActive) {
payload = captureFromCamera() || payload;
}
if (!payload) return;
// Keep the existing workflow contract: the accompanying text stays the
// "【拍摄完成】" marker that advances the FastGPT camera step; the image is
// the new multimodal attachment.
if (!sendImage(payload, CAMERA_DONE_TEXT)) return;
resetCameraInput();
}
/* --------------------------------------------------------- Chat updates */
function handleUserTranscript(text) {
@@ -864,7 +1191,6 @@ function handleAssistantFinal(text, interrupted) {
if (interrupted) {
state.currentAssistantBubble.classList.add("bubble--interrupted");
}
updateCameraQuestion(text);
state.currentAssistantBubble = null;
scrollChatToBottom();
}
@@ -930,16 +1256,16 @@ async function connect() {
const chatId = inputChatId || generateChatId();
const url = wsUrlWithChatId(chatId);
if (!url) {
setStatus("error", "Missing URL");
setStatus("error", "缺少服务器地址");
return;
}
state.connecting = true;
state.chatId = chatId;
els.chatId.value = chatId;
setStatus("connecting", "Connecting…");
setStatus("connecting", "连接中…");
setConnectButton();
addWsLog("system", `connecting ${url}`);
addWsLog("system", `正在连接 ${url}`);
try {
// Pre-warm audio context on user gesture so playback works on Safari.
@@ -949,9 +1275,9 @@ async function connect() {
state.connecting = false;
state.chatId = "";
if (!inputChatId) els.chatId.value = "";
setStatus("error", "Audio init failed");
setStatus("error", "音频初始化失败");
setConnectButton();
addWsLog("error", `audio init failed: ${err.message || err}`, "error");
addWsLog("error", `音频初始化失败:${err.message || err}`, "error");
return;
}
@@ -963,9 +1289,9 @@ async function connect() {
state.connecting = false;
state.chatId = "";
if (!inputChatId) els.chatId.value = "";
setStatus("error", "Bad URL");
setStatus("error", "服务器地址无效");
setConnectButton();
addWsLog("error", `bad websocket URL: ${err.message || err}`, "error");
addWsLog("error", `WebSocket 地址无效:${err.message || err}`, "error");
return;
}
ws.binaryType = "arraybuffer";
@@ -986,15 +1312,15 @@ async function connect() {
state.connecting = false;
state.connected = true;
resetPlaybackClock();
addWsLog("system", "websocket open");
setStatus("connected", "Connected");
addWsLog("system", "连接已建立");
setStatus("connected", "已连接");
setConnectButton();
setMicButton();
setMicSelectEnabled();
refreshMicDevices();
wsSend(JSON.stringify(startMessage));
addBubble("system", "Session started.");
addBubble("system", "会话已开始。");
setComposerEnabled(true);
setCameraButtonEnabled();
els.textInput.focus();
@@ -1026,7 +1352,7 @@ async function connect() {
ws.addEventListener("error", (err) => {
console.error("WebSocket error", err);
setStatus("error", "Connection error");
setStatus("error", "连接错误");
addWsLog("error", "websocket error", "error");
});
@@ -1055,11 +1381,11 @@ async function connect() {
if (wasConnected) {
addBubble(
"system",
`Session ended${event.reason ? `${event.reason}` : ""}.`,
`会话已结束${event.reason ? `${event.reason}` : ""}`,
);
setStatus("idle", "Disconnected");
setStatus("idle", "未连接");
} else {
setStatus("error", "Connection closed");
setStatus("error", "连接已断开");
}
});
}
@@ -1101,7 +1427,7 @@ els.micBtn.addEventListener("click", async () => {
}
} catch (err) {
console.error("Mic error", err);
addBubble("system", `Mic error: ${err.message || err}`);
addBubble("system", `麦克风错误:${err.message || err}`);
} finally {
els.micBtn.disabled = !state.connected;
}
@@ -1118,7 +1444,7 @@ els.micSelect.addEventListener("change", async () => {
await startMic();
} catch (err) {
console.error("Mic switch error", err);
addBubble("system", `Mic switch error: ${err.message || err}`);
addBubble("system", `麦克风切换错误:${err.message || err}`);
} finally {
setMicButton();
setMicSelectEnabled();
@@ -1139,7 +1465,25 @@ els.clearWsLogBtn.addEventListener("click", () => {
els.cameraDoneBtn.addEventListener("click", () => {
if (!state.cameraState) return;
sendText(CAMERA_DONE_TEXT);
submitCameraImage();
});
els.cameraStartBtn.addEventListener("click", () => {
startCamera(els.cameraDeviceSelect.value || undefined);
});
els.cameraDeviceSelect.addEventListener("change", () => {
// Switching device only restarts the stream when the camera is already live;
// otherwise the choice is applied when "使用摄像头" is pressed.
if (state.cameraActive) {
startCamera(els.cameraDeviceSelect.value || undefined);
}
});
els.cameraUpload.addEventListener("change", (event) => {
const file = event.target.files && event.target.files[0];
selectFileImage(file);
event.target.value = "";
});
function autosizeTextarea() {
@@ -1174,6 +1518,7 @@ els.textInput.addEventListener("keydown", (event) => {
});
window.addEventListener("beforeunload", () => {
stopCameraStream();
if (state.ws) {
try {
state.ws.close();
@@ -1192,7 +1537,7 @@ window.addEventListener("beforeunload", () => {
els.url.value = defaultWsUrl();
setStatus("idle", "Disconnected");
setStatus("idle", "未连接");
setConnectButton();
setMicButton();
setMicSelectEnabled();