From 4601aeab7578ac659192cff4e1c4ec17e6fffb09 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 9 Jul 2026 08:41:57 +0800 Subject: [PATCH] Add AI Element2 Gated Copilot script for enhanced media interaction - Introduce a new user script `copilot-element2-two-party.js` that implements a two-party gated bridge for audio input to AI, ensuring both sides are connected before processing. - Implement functionality for managing audio and video tracks, including recording capabilities for AI input. - Enhance chat logging features to provide real-time feedback on AI interactions and recording status. - Integrate mechanisms for handling media streams, improving overall media management in the WebRTC environment. --- scripts/copilot-element2-two-party.js | 1218 +++++++++++++++++++++++++ 1 file changed, 1218 insertions(+) create mode 100644 scripts/copilot-element2-two-party.js diff --git a/scripts/copilot-element2-two-party.js b/scripts/copilot-element2-two-party.js new file mode 100644 index 0000000..becea8a --- /dev/null +++ b/scripts/copilot-element2-two-party.js @@ -0,0 +1,1218 @@ +// ==UserScript== +// @name 12345 AI Element2 Gated Copilot +// @namespace http://tampermonkey.net/ +// @version 1.0 +// @description Two-party gated bridge: only feed element-2:audio into AI after both sides are connected. +// @author You +// @match *://yhy-yw.22sj.cn:*/* +// @grant none +// @run-at document-start +// ==/UserScript== + +(function () { + 'use strict'; + + const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; + const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio'; + const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video'; + + const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; + const originalAddTrack = RTCPeerConnection.prototype.addTrack; + const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; + const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; + const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); + + const remoteAudio = new Map(); + const remoteVideo = new Map(); + const businessSenders = new Map(); + const aiOutputTrackIds = new Set(); + const inboundMemory = new Map(); + const outboundMemory = new Map(); + const internalPcs = new WeakSet(); + const AUDIO_UPLINK_STORAGE_KEY = 'element2-copilot-send-audio'; + const VIDEO_UPLINK_STORAGE_KEY = 'element2-copilot-send-video'; + + let selectedRemoteAudio = null; + let selectedRemoteVideo = null; + let selectedBusinessSender = null; + let originalBusinessTrack = null; + let aiAudioTrack = null; + + let aiPc = null; + let aiWs = null; + let aiDc = null; + let aiAudioSender = null; + let aiVideoSender = null; + let isAiTakeover = false; + let aiAttached = false; + let inputVu = 0; + let outputVu = 0; + let inputHealthTimer = null; + let outputHealthTimer = null; + let assistantLine = null; + let assistantBuffer = ''; + let inputRecorder = null; + let inputRecordingChunks = []; + let inputRecordingTrack = null; + let inputRecordingStartedAt = 0; + let videoRecorder = null; + let videoRecordingChunks = []; + let videoRecordingTrack = null; + let videoRecordingStartedAt = 0; + + function log(message) { + console.log(`[AI Element2 Gated] ${message}`); + } + + function readBoolSetting(key, fallback) { + try { + const value = localStorage.getItem(key); + if (value === null) return fallback; + return value === 'true'; + } catch { + return fallback; + } + } + + function writeBoolSetting(key, value) { + try { + localStorage.setItem(key, value ? 'true' : 'false'); + } catch { + // Ignore storage failures; the current checkbox state still applies. + } + } + + function shouldSendAudioToAi() { + return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); + } + + function shouldSendVideoToAi() { + return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); + } + + function appendChatLine(role, text, important = false) { + const box = document.getElementById('element2-copilot-log'); + if (!box) return; + if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; + const row = document.createElement('div'); + row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; + row.style.marginBottom = '4px'; + if (important) row.style.color = '#fcd34d'; + box.appendChild(row); + while (box.children.length > 120) box.removeChild(box.firstChild); + box.scrollTop = box.scrollHeight; + return row; + } + + function startAssistantLine() { + assistantBuffer = ''; + assistantLine = null; + } + + function appendAssistantDelta(delta) { + if (!delta) return; + const box = document.getElementById('element2-copilot-log'); + if (!box) { + console.log(`[AI Element2 Gated] AI: ${delta}`); + return; + } + if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; + if (!assistantLine) assistantLine = appendChatLine('AI', ''); + assistantBuffer += delta; + assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; + box.scrollTop = box.scrollHeight; + } + + function finishAssistantLine() { + assistantLine = null; + assistantBuffer = ''; + } + + function logUserTranscript(text) { + if (!text?.trim()) return; + finishAssistantLine(); + appendChatLine('用户', text.trim(), true); + } + + function recorderMimeType() { + if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; + if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; + return ''; + } + + function recordingFileName(track) { + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; + return `ai-input-${trackPart}-${stamp}.webm`; + } + + function videoRecordingFileName(track) { + const stamp = new Date().toISOString().replace(/[:.]/g, '-'); + const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; + return `ai-video-${trackPart}-${stamp}.webm`; + } + + function videoRecorderMimeType() { + if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8'; + if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm'; + return ''; + } + + async function startInputRecording() { + if (inputRecorder?.state === 'recording') return; + selectRemoteAudio('record'); + if (!isLive(selectedRemoteAudio)) { + appendChatLine('录音', `没有可录制的 ${STATIC_AUDIO_ELEMENT_REASON} 音轨`, true); + return; + } + if (!window.MediaRecorder) { + appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); + return; + } + + inputRecordingChunks = []; + inputRecordingTrack = selectedRemoteAudio; + inputRecordingStartedAt = Date.now(); + const stream = new MediaStream([selectedRemoteAudio]); + const mimeType = recorderMimeType(); + inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + inputRecorder.ondataavailable = (event) => { + if (event.data?.size > 0) inputRecordingChunks.push(event.data); + }; + inputRecorder.onstop = () => { + const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = recordingFileName(inputRecordingTrack); + document.body.appendChild(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 30000); + + const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); + appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); + inputRecorder = null; + inputRecordingChunks = []; + inputRecordingTrack = null; + updateRecordButton(); + }; + inputRecorder.start(250); + appendChatLine('录音', `开始录制 AI 输入：${trackLabel(inputRecordingTrack)}`, true); + updateRecordButton(); + } + + function stopInputRecording() { + if (inputRecorder?.state === 'recording') inputRecorder.stop(); + } + + function toggleInputRecording() { + if (inputRecorder?.state === 'recording') stopInputRecording(); + else startInputRecording(); + } + + function updateRecordButton() { + const button = document.getElementById('element2-copilot-record'); + if (!button) return; + const recording = inputRecorder?.state === 'recording'; + button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; + button.style.background = recording ? '#dc2626' : '#2563eb'; + } + + function updateVideoRecordButton() { + const button = document.getElementById('element2-copilot-record-video'); + if (!button) return; + const recording = videoRecorder?.state === 'recording'; + button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频'; + button.style.background = recording ? '#dc2626' : '#475569'; + } + + async function startVideoRecording() { + if (videoRecorder?.state === 'recording') return; + selectRemoteVideo('record-video'); + if (!isLive(selectedRemoteVideo)) { + appendChatLine('录制视频', `没有可录制的 ${STATIC_VIDEO_ELEMENT_REASON} 视频轨`, true); + return; + } + if (!window.MediaRecorder) { + appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true); + return; + } + + videoRecordingChunks = []; + videoRecordingTrack = selectedRemoteVideo; + videoRecordingStartedAt = Date.now(); + const stream = new MediaStream([videoRecordingTrack]); + const mimeType = videoRecorderMimeType(); + videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); + videoRecorder.ondataavailable = (event) => { + if (event.data?.size > 0) videoRecordingChunks.push(event.data); + }; + videoRecorder.onstop = () => { + const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' }); + const url = URL.createObjectURL(blob); + const link = document.createElement('a'); + link.href = url; + link.download = videoRecordingFileName(videoRecordingTrack); + document.body.appendChild(link); + link.click(); + link.remove(); + setTimeout(() => URL.revokeObjectURL(url), 30000); + + const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1); + appendChatLine('录制视频', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); + videoRecorder = null; + videoRecordingChunks = []; + videoRecordingTrack = null; + updateVideoRecordButton(); + }; + videoRecorder.start(250); + appendChatLine('录制视频', `开始录制 AI 视频：${trackLabel(videoRecordingTrack)}`, true); + updateVideoRecordButton(); + } + + function stopVideoRecording() { + if (videoRecorder?.state === 'recording') videoRecorder.stop(); + } + + function toggleVideoRecording() { + if (videoRecorder?.state === 'recording') stopVideoRecording(); + else startVideoRecording(); + } + + function pcId() { + const bytes = new Uint8Array(16); + crypto.getRandomValues(bytes); + return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); + } + + function isInternalPc(pc) { + return Boolean(pc && (pc.__aiElement2GatedInternal || internalPcs.has(pc) || pc === aiPc)); + } + + function isLive(track) { + return track && track.readyState === 'live'; + } + + function trackLabel(track) { + if (!track) return 'none'; + return `${track.id}${track.muted ? ' muted' : ''}`; + } + + function markInternalPc(pc) { + if (!pc) return; + pc.__aiElement2GatedInternal = true; + internalPcs.add(pc); + } + + function elementReasonFor(element) { + if (!element) return null; + const tag = element.tagName?.toLowerCase(); + if (tag !== 'audio' && tag !== 'video') return null; + const elements = Array.from(document.querySelectorAll(tag)); + const index = elements.indexOf(element); + if (index < 0) return `${tag}-srcObject`; + return `element-${index}:${tag}`; + } + + function hasElementReason(item, reason) { + return Boolean(item?.reasons?.has(reason)); + } + + function elementAudioReasons(item) { + return Array.from(item?.reasons || []).filter((reason) => /^element-\d+:audio$/.test(reason)); + } + + function audioSourceLabel(item) { + const reasons = elementAudioReasons(item); + return reasons[0] || 'audio-track'; + } + + function updateSelectedAudioLabel() { + const label = document.getElementById('element2-copilot-selected-audio'); + if (!label) return; + const item = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); + label.textContent = item + ? `${audioSourceLabel(item)} ${selectedRemoteAudio.id.slice(0, 8)} score=${Math.round(item.audioScore || 0)}` + : `等待 ${STATIC_AUDIO_ELEMENT_REASON}`; + } + + function pageHasBothParties() { + const text = document.body?.innerText || ''; + return text.includes('当事人') && text.includes('坐席端'); + } + + function staticAudioItem() { + return Array.from(remoteAudio.values()).find((candidate) => ( + isLive(candidate.track) + && !candidate.track.muted + && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON) + )) || null; + } + + function gateStatus() { + const item = staticAudioItem(); + return { + bothParties: pageHasBothParties(), + audioReady: Boolean(item), + senderReady: Boolean(selectedBusinessSender), + track: item?.track || null, + }; + } + + function gateMessage(status = gateStatus()) { + if (!status.bothParties) return '等待当事人和坐席端同时在线'; + if (!status.audioReady) return `等待 ${STATIC_AUDIO_ELEMENT_REASON} 有声轨`; + if (!status.senderReady) return '等待业务上行 sender'; + return `就绪：${STATIC_AUDIO_ELEMENT_REASON} -> AI`; + } + + function updateGateLabel() { + const label = document.getElementById('element2-copilot-gate'); + if (!label) return; + const status = gateStatus(); + label.textContent = gateMessage(status); + label.style.color = status.bothParties && status.audioReady && status.senderReady ? '#86efac' : '#fcd34d'; + } + + function scoreAudioCandidate(item) { + if (!item || !isLive(item.track) || item.track.muted || aiOutputTrackIds.has(item.track.id)) return -1; + let score = 0; + if (item.receiver) score += 300; + if ((item.bytesReceived || 0) > 0) score += 100; + if ((item.bytesDelta || 0) > 0) score += 4000 + Math.min(1500, item.bytesDelta); + if ((item.packetsDelta || 0) > 0) score += 2000 + Math.min(800, item.packetsDelta * 8); + score += Math.min(1200, (item.audioLevel || 0) * 12000); + if (elementAudioReasons(item).length > 0) score += 100; + item.audioScore = score; + return score; + } + + function rememberRemoteReceiver(receiver, reason) { + const track = receiver?.track; + if (!isLive(track) || isInternalPc(receiver?.transport)) return; + + if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) { + let item = remoteAudio.get(track.id); + if (!item) { + item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 }; + remoteAudio.set(track.id, item); + log(`audio candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.receiver = receiver || item.receiver; + item.reasons.add(reason); + selectRemoteAudio(reason); + } + + if (track.kind === 'video') { + let item = remoteVideo.get(track.id); + if (!item) { + item = { track, receiver, reasons: new Set() }; + remoteVideo.set(track.id, item); + log(`video candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.receiver = receiver || item.receiver; + item.reasons.add(reason); + selectRemoteVideo(reason); + } + } + + function rememberRemoteStream(stream, reason) { + if (!stream || !(stream instanceof MediaStream)) return; + for (const track of stream.getAudioTracks()) { + if (aiOutputTrackIds.has(track.id)) continue; + let item = remoteAudio.get(track.id); + if (!item) { + item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 }; + remoteAudio.set(track.id, item); + log(`audio stream candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.reasons.add(reason); + } + for (const track of stream.getVideoTracks()) { + let item = remoteVideo.get(track.id); + if (!item) { + item = { track, receiver: null, reasons: new Set() }; + remoteVideo.set(track.id, item); + log(`video stream candidate ${trackLabel(track)} (${reason})`, true); + } + item.track = track; + item.reasons.add(reason); + } + selectRemoteAudio(reason); + selectRemoteVideo(reason); + } + + function selectRemoteAudio(reason) { + const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); + if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted && hasElementReason(current, STATIC_AUDIO_ELEMENT_REASON)) { + updateGateLabel(); + return selectedRemoteAudio; + } + + const item = staticAudioItem(); + if (!item) { + updateSelectedAudioLabel(); + updateGateLabel(); + return null; + } + + if (selectedRemoteAudio !== item.track) { + selectedRemoteAudio = item.track; + log(`selected input audio (${reason}) from ${STATIC_AUDIO_ELEMENT_REASON}: ${trackLabel(selectedRemoteAudio)}`, true); + updateSelectedAudioLabel(); + attachAudioToAi(); + } + updateGateLabel(); + return selectedRemoteAudio; + } + + function selectRemoteVideo(reason) { + const current = selectedRemoteVideo && remoteVideo.get(selectedRemoteVideo.id); + if (isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted && hasElementReason(current, STATIC_VIDEO_ELEMENT_REASON)) { + return selectedRemoteVideo; + } + + const item = Array.from(remoteVideo.values()).find((candidate) => ( + isLive(candidate.track) + && !candidate.track.muted + && hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON) + )); + if (!item) return null; + + if (selectedRemoteVideo !== item.track) { + selectedRemoteVideo = item.track; + log(`selected input video (${reason}) from ${STATIC_VIDEO_ELEMENT_REASON}: ${trackLabel(selectedRemoteVideo)}`, true); + attachVideoToAi(); + } + return selectedRemoteVideo; + } + + function rememberBusinessSender(sender, track, reason) { + if (!sender) return; + const senderTrack = track || sender.track; + if (senderTrack && senderTrack.kind !== 'audio') return; + + const id = sender.__aiElement2GatedSenderId || `sender-${businessSenders.size + 1}`; + sender.__aiElement2GatedSenderId = id; + + let item = businessSenders.get(id); + if (!item) { + item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 }; + businessSenders.set(id, item); + log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true); + } + item.sender = sender; + item.track = sender.track || senderTrack || item.track; + if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack; + item.sources.add(reason); + wrapBusinessSender(sender); + selectBusinessSender(reason); + } + + function wrapBusinessSender(sender) { + if (!sender || sender.__aiElement2GatedWrapped) return; + const originalReplaceTrack = sender.replaceTrack?.bind(sender); + if (!originalReplaceTrack) return; + + sender.replaceTrack = async function (track) { + const result = await originalReplaceTrack(track); + rememberBusinessSender(sender, track, 'replaceTrack'); + return result; + }; + sender.__aiElement2GatedWrapped = true; + } + + function isRemoteInputTrack(track) { + return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id)); + } + + function selectBusinessSender(reason) { + if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender; + + const item = Array.from(businessSenders.values()).find((candidate) => { + const track = candidate.sender.track || candidate.track || candidate.originalTrack; + return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track); + }); + if (!item) return null; + + selectedBusinessSender = item.sender; + originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; + log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); + updateGateLabel(); + attachAiOutputToBusiness(); + return selectedBusinessSender; + } + + function setVu(id, level) { + const el = document.getElementById(id); + if (el) el.style.width = `${Math.min(100, level * 2500)}%`; + } + + function renderVu() { + inputVu *= 0.88; + outputVu *= 0.88; + setVu('element2-copilot-input-vu', inputVu); + setVu('element2-copilot-output-vu', outputVu); + requestAnimationFrame(renderVu); + } + + async function refreshStats() { + await refreshInputStats(); + await refreshOutputStats(); + } + + async function refreshInputStats() { + for (const item of remoteAudio.values()) { + if (!item.receiver?.getStats) continue; + const report = await item.receiver.getStats(); + report.forEach((stat) => { + const kind = stat.kind || stat.mediaType; + if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; + const prev = inboundMemory.get(stat.id); + const bytes = stat.bytesReceived || 0; + const packets = stat.packetsReceived || 0; + let audioLevel = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0; + if ( + prev + && typeof stat.totalAudioEnergy === 'number' + && typeof stat.totalSamplesDuration === 'number' + && typeof prev.energy === 'number' + && typeof prev.duration === 'number' + ) { + const energyDelta = stat.totalAudioEnergy - prev.energy; + const durationDelta = stat.totalSamplesDuration - prev.duration; + if (energyDelta > 0 && durationDelta > 0) { + audioLevel = Math.max(audioLevel, Math.sqrt(energyDelta / durationDelta)); + } + } + item.bytesReceived = bytes; + item.packetsReceived = packets; + item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; + item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; + item.audioLevel = audioLevel; + inboundMemory.set(stat.id, { + bytes, + packets, + energy: stat.totalAudioEnergy, + duration: stat.totalSamplesDuration, + }); + if (item.track === selectedRemoteAudio) { + inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500), Math.min(0.18, audioLevel)); + } + }); + } + selectRemoteAudio('stats'); + } + + async function refreshOutputStats() { + for (const item of businessSenders.values()) { + if (!item.sender?.getStats) continue; + item.track = item.sender.track || item.track; + const report = await item.sender.getStats(); + report.forEach((stat) => { + const kind = stat.kind || stat.mediaType; + if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; + const key = `${item.id}:${stat.id}`; + const prev = outboundMemory.get(key); + const bytes = stat.bytesSent || 0; + const packets = stat.packetsSent || 0; + item.bytesSent = bytes; + item.packetsSent = packets; + item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; + item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; + outboundMemory.set(key, { bytes, packets }); + if (item.sender === selectedBusinessSender) { + outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); + } + }); + } + } + + async function outboundBytes(sender) { + if (!sender?.getStats) return null; + let total = 0; + const report = await sender.getStats(); + report.forEach((stat) => { + const kind = stat.kind || stat.mediaType; + if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0; + }); + return total; + } + + function aiState() { + if (!aiPc) return 'pc=null'; + return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; + } + + function aiReady() { + return Boolean(aiPc?.remoteDescription && ( + aiPc.connectionState === 'connected' + || aiPc.iceConnectionState === 'connected' + || aiPc.iceConnectionState === 'completed' + )); + } + + function waitForAiReady(timeoutMs = 10000) { + if (aiReady()) return Promise.resolve(true); + return new Promise((resolve) => { + const startedAt = Date.now(); + const timer = setInterval(() => { + if (aiReady()) { + clearInterval(timer); + resolve(true); + } else if (Date.now() - startedAt >= timeoutMs) { + clearInterval(timer); + resolve(false); + } + }, 250); + }); + } + + function checkAiInput(track) { + if (inputHealthTimer) clearTimeout(inputHealthTimer); + inputHealthTimer = setTimeout(async () => { + if (!aiAudioSender || aiAudioSender.track !== track) return; + await waitForAiReady(); + const before = await outboundBytes(aiAudioSender); + await new Promise((resolve) => setTimeout(resolve, 2500)); + const after = await outboundBytes(aiAudioSender); + log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true); + }, 500); + } + + function checkAiOutput(track) { + if (outputHealthTimer) clearTimeout(outputHealthTimer); + outputHealthTimer = setTimeout(async () => { + if (!selectedBusinessSender || selectedBusinessSender.track !== track) return; + const before = await outboundBytes(selectedBusinessSender); + await new Promise((resolve) => setTimeout(resolve, 2500)); + const after = await outboundBytes(selectedBusinessSender); + log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true); + }, 800); + } + + async function attachAudioToAi() { + if (!shouldSendAudioToAi()) return false; + if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false; + if (aiAudioSender.track === selectedRemoteAudio) return true; + await aiAudioSender.replaceTrack(selectedRemoteAudio); + log(`AI input uses ${trackLabel(selectedRemoteAudio)} from ${STATIC_AUDIO_ELEMENT_REASON}`, true); + checkAiInput(selectedRemoteAudio); + return true; + } + + async function attachVideoToAi() { + if (!shouldSendVideoToAi()) return false; + if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false; + if (aiVideoSender.track === selectedRemoteVideo) return true; + await aiVideoSender.replaceTrack(selectedRemoteVideo); + log(`AI video uses ${trackLabel(selectedRemoteVideo)} from ${STATIC_VIDEO_ELEMENT_REASON}`, true); + return true; + } + + async function attachAiOutputToBusiness() { + selectBusinessSender('before-output'); + if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) { + log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`); + return false; + } + if (selectedBusinessSender.track === aiAudioTrack) return true; + await selectedBusinessSender.replaceTrack(aiAudioTrack); + aiAttached = true; + log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true); + checkAiOutput(aiAudioTrack); + return true; + } + + async function restoreBusinessAudio() { + if (!selectedBusinessSender || !aiAttached) return; + await selectedBusinessSender.replaceTrack(originalBusinessTrack || null); + aiAttached = false; + log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true); + } + + function wrapTrackListener(listener) { + if (!listener || listener.__aiElement2GatedWrapped) return listener; + const wrapped = function (event) { + const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget; + if (!isInternalPc(ownerPc) && event?.track) { + rememberRemoteReceiver(event.receiver, 'pc-track'); + for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); + } + if (typeof listener === 'function') return listener.apply(this, arguments); + if (listener?.handleEvent) return listener.handleEvent(event); + return undefined; + }; + wrapped.__aiElement2GatedWrapped = true; + return wrapped; + } + + RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { + if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); + return originalAddEventListener.call(this, type, listener, options); + }; + + try { + const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); + Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { + configurable: true, + enumerable: true, + get() { + return descriptor?.get ? descriptor.get.call(this) : this.__aiElement2GatedOnTrack || null; + }, + set(listener) { + const wrapped = wrapTrackListener(listener); + this.__aiElement2GatedOnTrack = wrapped; + if (descriptor?.set) descriptor.set.call(this, wrapped); + else this.addEventListener('track', wrapped); + }, + }); + } catch (error) { + console.warn('[AI Element2 Gated] failed to wrap ontrack', error); + } + + RTCPeerConnection.prototype.addTrack = function (track, ...streams) { + const sender = originalAddTrack.call(this, track, ...streams); + if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack'); + return sender; + }; + + if (originalAddTransceiver) { + RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { + const transceiver = originalAddTransceiver.call(this, trackOrKind, init); + if (!isInternalPc(this)) { + const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; + if (kind === 'audio') { + const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; + rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver'); + } + } + return transceiver; + }; + } + + RTCPeerConnection.prototype.setRemoteDescription = function () { + const result = originalSetRemoteDescription.apply(this, arguments); + if (!isInternalPc(this)) { + Promise.resolve(result).then(() => { + for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd'); + for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd'); + }).catch(() => {}); + } + return result; + }; + + if (mediaSrcObjectDescriptor?.set) { + Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { + configurable: true, + enumerable: mediaSrcObjectDescriptor.enumerable, + get() { + return mediaSrcObjectDescriptor.get.call(this); + }, + set(stream) { + if (!this.__aiElement2GatedCopilotElement) { + const reason = elementReasonFor(this) || `${this.tagName.toLowerCase()}-srcObject`; + rememberRemoteStream(stream, reason); + } + return mediaSrcObjectDescriptor.set.call(this, stream); + }, + }); + } + + function scanMediaElements() { + document.querySelectorAll('audio').forEach((element, index) => { + if (element.__aiElement2GatedCopilotElement) return; + rememberRemoteStream(element.srcObject, `element-${index}:audio`); + }); + document.querySelectorAll('video').forEach((element, index) => { + if (element.__aiElement2GatedCopilotElement) return; + rememberRemoteStream(element.srcObject, `element-${index}:video`); + }); + selectRemoteAudio('media-scan'); + selectRemoteVideo('media-scan'); + updateGateLabel(); + } + + setInterval(scanMediaElements, 1000); + + async function connectAi(signalingUrl, assistantId) { + scanMediaElements(); + selectRemoteAudio('connect'); + selectRemoteVideo('connect'); + selectBusinessSender('connect'); + const status = gateStatus(); + updateGateLabel(); + if (!status.bothParties || !status.audioReady || !status.senderReady) { + appendChatLine('门槛', gateMessage(status), true); + setStatus('disconnected'); + return false; + } + + disconnectAi(); + selectedRemoteAudio = status.track; + + const id = pcId(); + aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); + markInternalPc(aiPc); + aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); + aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); + + aiDc = aiPc.createDataChannel('chat'); + aiDc.onopen = () => { + setStatus('connected'); + log('AI data channel open', true); + aiDc.send(JSON.stringify({ type: 'client-ready' })); + }; + aiDc.onclose = () => setStatus('disconnected'); + aiDc.onmessage = (event) => { + try { + const msg = JSON.parse(event.data); + const content = msg.delta || msg.content; + if (msg.type === 'transcript') { + logUserTranscript(content); + } else if (msg.type === 'assistant-text-start') { + startAssistantLine(); + } else if (msg.type === 'assistant-text-delta') { + appendAssistantDelta(content); + } else if (msg.type === 'assistant-text-end') { + finishAssistantLine(); + } else { + log(`data channel message: ${msg.type}`); + } + } catch { + appendChatLine('DATA', event.data); + } + }; + + const sendAudio = shouldSendAudioToAi(); + const audioTransceiver = sendAudio && isLive(selectedRemoteAudio) + ? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' }) + : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); + aiAudioSender = audioTransceiver.sender; + log(!sendAudio + ? 'AI audio uplink disabled by panel setting' + : isLive(selectedRemoteAudio) + ? `AI input starts with ${trackLabel(selectedRemoteAudio)} (${STATIC_AUDIO_ELEMENT_REASON})` + : `AI input waiting for ${STATIC_AUDIO_ELEMENT_REASON}`, true); + + const sendVideo = shouldSendVideoToAi(); + const shouldSendVideo = sendVideo && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; + if (shouldSendVideo) { + const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' }); + aiVideoSender = videoTransceiver.sender; + log(`AI video starts with ${trackLabel(selectedRemoteVideo)} (${STATIC_VIDEO_ELEMENT_REASON})`); + } else { + aiVideoSender = null; + log(sendVideo + ? `AI video disabled: waiting for live track at ${STATIC_VIDEO_ELEMENT_REASON}` + : 'AI video uplink disabled by panel setting'); + } + + aiPc.ontrack = (event) => { + if (event.track.kind !== 'audio') return; + aiOutputTrackIds.add(event.track.id); + remoteAudio.delete(event.track.id); + aiAudioTrack = event.track; + log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); + aiAudioTrack.onunmute = () => { + log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); + attachAiOutputToBusiness(); + }; + attachAiOutputToBusiness(); + }; + + aiPc.onicecandidate = (event) => { + if (aiWs?.readyState !== WebSocket.OPEN) return; + aiWs.send(JSON.stringify({ + type: 'ice-candidate', + payload: { + pc_id: id, + candidate: event.candidate ? { + candidate: event.candidate.candidate, + sdpMid: event.candidate.sdpMid, + sdpMLineIndex: event.candidate.sdpMLineIndex, + } : null, + }, + })); + }; + + setStatus('connecting'); + aiWs = new WebSocket(signalingUrl); + aiWs.onopen = async () => { + const offer = await aiPc.createOffer(); + await aiPc.setLocalDescription(offer); + aiWs.send(JSON.stringify({ + type: 'offer', + payload: { + pc_id: id, + sdp: aiPc.localDescription.sdp, + type: aiPc.localDescription.type, + assistant_id: assistantId, + vision_enabled: shouldSendVideo, + }, + })); + log('sent AI WebRTC offer', true); + }; + aiWs.onmessage = async (event) => { + try { + const msg = JSON.parse(event.data); + if (msg.type === 'answer') { + await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); + log(`AI WebRTC answer applied; ${aiState()}`, true); + } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { + await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); + } else if (msg.type === 'error') { + throw new Error(msg.payload?.message || 'server error'); + } + } catch (error) { + log(`signaling error: ${error.message || error}`); + } + }; + aiWs.onerror = () => { + setStatus('failed'); + log('AI signaling websocket error'); + }; + aiWs.onclose = () => setStatus('disconnected'); + + attachAudioToAi(); + attachVideoToAi(); + return true; + } + + function disconnectAi() { + try { aiDc?.close(); } catch {} + try { aiPc?.close(); } catch {} + try { aiWs?.close(); } catch {} + aiDc = null; + aiPc = null; + aiWs = null; + aiAudioSender = null; + aiVideoSender = null; + aiAudioTrack = null; + if (inputRecorder?.state === 'recording') stopInputRecording(); + if (videoRecorder?.state === 'recording') stopVideoRecording(); + setStatus('disconnected'); + } + + function setStatus(status) { + const el = document.getElementById('element2-copilot-status'); + if (!el) return; + el.textContent = { + disconnected: '未连接', + connecting: '连接中', + connected: '已连接', + failed: '连接失败', + }[status] || status; + } + + function updateModeButton() { + const button = document.getElementById('element2-copilot-toggle'); + if (!button) return; + if (isAiTakeover) { + button.textContent = '当前：AI 接管｜点击切回人工'; + button.style.background = '#8b5cf6'; + } else { + button.textContent = '当前：人工接管｜点击切换 AI'; + button.style.background = '#10b981'; + } + } + + window.__aiElement2GatedDump = function () { + const status = gateStatus(); + return { + audioSelection: STATIC_AUDIO_ELEMENT_REASON, + staticVideoElement: STATIC_VIDEO_ELEMENT_REASON, + gate: { + bothParties: status.bothParties, + audioReady: status.audioReady, + senderReady: status.senderReady, + track: trackLabel(status.track), + }, + gateMessage: gateMessage(status), + selectedInput: trackLabel(selectedRemoteAudio), + selectedVideo: trackLabel(selectedRemoteVideo), + aiVideoSenderTrack: trackLabel(aiVideoSender?.track), + selectedOutput: trackLabel(selectedBusinessSender?.track), + originalOutput: trackLabel(originalBusinessTrack), + aiAudio: trackLabel(aiAudioTrack), + aiAttached, + aiState: aiState(), + recording: inputRecorder?.state === 'recording' ? { + track: trackLabel(inputRecordingTrack), + seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), + chunks: inputRecordingChunks.length, + } : null, + videoRecording: videoRecorder?.state === 'recording' ? { + track: trackLabel(videoRecordingTrack), + seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1), + chunks: videoRecordingChunks.length, + } : null, + inputs: Array.from(remoteAudio.values()).map((item) => ({ + selected: item.track === selectedRemoteAudio, + id: item.track.id, + muted: item.track.muted, + live: item.track.readyState, + hasReceiver: Boolean(item.receiver), + source: audioSourceLabel(item), + bytesReceived: item.bytesReceived || 0, + packetsReceived: item.packetsReceived || 0, + bytesDelta: item.bytesDelta, + packetsDelta: item.packetsDelta, + audioLevel: item.audioLevel || 0, + audioScore: Math.round(item.audioScore || scoreAudioCandidate(item) || 0), + matchesStaticElement: hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON), + reasons: Array.from(item.reasons), + })), + videos: Array.from(remoteVideo.values()).map((item) => ({ + selected: item.track === selectedRemoteVideo, + id: item.track.id, + muted: item.track.muted, + live: item.track.readyState, + hasReceiver: Boolean(item.receiver), + matchesStaticElement: hasElementReason(item, STATIC_VIDEO_ELEMENT_REASON), + reasons: Array.from(item.reasons), + })), + outputs: Array.from(businessSenders.values()).map((item) => ({ + selected: item.sender === selectedBusinessSender, + id: item.id, + track: trackLabel(item.sender.track || item.track), + originalTrack: trackLabel(item.originalTrack), + pcAddTrack: item.sources.has('pc-addTrack'), + isRemoteInput: isRemoteInputTrack(item.sender.track || item.track), + bytesSent: item.bytesSent, + packetsSent: item.packetsSent, + bytesDelta: item.bytesDelta, + packetsDelta: item.packetsDelta, + sources: Array.from(item.sources), + })), + }; + }; + + function renderPanel() { + if (!document.body || document.getElementById('element2-copilot-panel')) return; + const panel = document.createElement('div'); + panel.id = 'element2-copilot-panel'; + panel.style.cssText = [ + 'position:fixed', + 'top:20px', + 'left:20px', + 'width:360px', + 'z-index:999999', + 'box-sizing:border-box', + 'padding:16px', + 'border-radius:10px', + 'background:rgba(17,24,39,.96)', + 'color:white', + 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', + 'box-shadow:0 12px 32px rgba(0,0,0,.35)', + ].join(';'); + panel.innerHTML = ` +
+ AI 元素选轨 + 未连接 +
+
+ 音频输入：${STATIC_AUDIO_ELEMENT_REASON}
+ 视频输入：${STATIC_VIDEO_ELEMENT_REASON}
+ 等待当事人和坐席端同时在线 +
+ + + + +
+ + +
+
+
等待活跃音轨原生音轨
+
+
+
+
AI 输出业务上行
+
+
+ +
+ + +
+
+ 等待交互... +
+ `; + document.body.appendChild(panel); + + const toggle = document.getElementById('element2-copilot-toggle'); + const recordButton = document.getElementById('element2-copilot-record'); + const videoRecordButton = document.getElementById('element2-copilot-record-video'); + const urlInput = document.getElementById('element2-copilot-url'); + const assistantInput = document.getElementById('element2-copilot-assistant'); + const sendAudioInput = document.getElementById('element2-copilot-send-audio'); + const sendVideoInput = document.getElementById('element2-copilot-send-video'); + sendAudioInput.checked = shouldSendAudioToAi(); + sendVideoInput.checked = shouldSendVideoToAi(); + sendAudioInput.addEventListener('change', () => { + writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked); + log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`); + }); + sendVideoInput.addEventListener('change', () => { + writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked); + log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`); + }); + recordButton.addEventListener('click', toggleInputRecording); + videoRecordButton.addEventListener('click', toggleVideoRecording); + toggle.addEventListener('click', async () => { + if (!isAiTakeover && !assistantInput.value.trim()) { + alert('请先填写助手 ID。'); + return; + } + isAiTakeover = !isAiTakeover; + updateModeButton(); + if (isAiTakeover) { + const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim()); + if (connected) { + await attachAiOutputToBusiness(); + } else { + isAiTakeover = false; + updateModeButton(); + } + } else { + await restoreBusinessAudio(); + disconnectAi(); + } + }); + updateModeButton(); + updateRecordButton(); + updateVideoRecordButton(); + updateGateLabel(); + } + + function renderPanelSoon() { + try { renderPanel(); } catch (error) { console.warn('[AI Element2 Gated] panel error', error); } + } + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true }); + } else { + renderPanelSoon(); + } + requestAnimationFrame(renderVu); + setInterval(refreshStats, 250); + setTimeout(renderPanelSoon, 1500); + setTimeout(renderPanelSoon, 3000); + window.addEventListener('beforeunload', () => { + if (inputRecorder?.state === 'recording') stopInputRecording(); + if (videoRecorder?.state === 'recording') stopVideoRecording(); + }); +})();