// ==UserScript== // @name 12345 AI Element2 Gated Copilot // @namespace http://tampermonkey.net/ // @version 1.0 // @description Two-party gated bridge: only feed participant audio into AI after both sides are connected. // @author You // @match *://yhy-yw.22sj.cn:*/* // @grant none // @run-at document-start // ==/UserScript== (function () { 'use strict'; const DEFAULT_API_BASE_URL = 'http://182.92.86.220:8000'; const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio'; const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video'; const PARTICIPANT_AUDIO_TRACK_PATTERN = /-audio-3-a$/; const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; const originalAddTrack = RTCPeerConnection.prototype.addTrack; const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); const remoteAudio = new Map(); const remoteVideo = new Map(); const businessSenders = new Map(); const aiOutputTrackIds = new Set(); const inboundMemory = new Map(); const outboundMemory = new Map(); const internalPcs = new WeakSet(); const AUDIO_UPLINK_STORAGE_KEY = 'element2-copilot-send-audio'; const VIDEO_UPLINK_STORAGE_KEY = 'element2-copilot-send-video'; const AUDIO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-audio-selection-mode'; const VIDEO_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-video-selection-mode'; const SENDER_SELECTION_MODE_STORAGE_KEY = 'element2-copilot-sender-selection-mode'; let selectedRemoteAudio = null; let audioSelectionMode = readAudioSelectionMode(); let manualAudioTrackId = null; let audioTrackSelectSignature = ''; let selectedRemoteVideo = null; let videoSelectionMode = readVideoSelectionMode(); let manualVideoTrackId = null; let videoTrackSelectSignature = ''; let selectedBusinessSender = null; let senderSelectionMode = readSenderSelectionMode(); let manualSenderId = null; let senderSelectSignature = ''; let changingBusinessSender = false; let originalBusinessTrack = null; let aiAudioTrack = null; let aiPc = null; let aiDc = null; let aiAudioSender = null; let aiVideoSender = null; let aiPcId = null; let aiOfferUrl = null; let aiAssistantId = null; let aiAuthorization = null; let aiCandidateQueue = []; let aiCandidateFlushTimer = null; let aiKeepAliveTimer = null; let aiNegotiating = false; let aiCanSendCandidates = false; let isAiTakeover = false; let aiAttached = false; let aiOutputAudioContext = null; let aiOutputAnalyser = null; let aiOutputAnalyserData = null; let aiOutputAnalyserSource = null; let aiOutputAnalyserTrack = null; let inputVu = 0; let outputVu = 0; let inputHealthTimer = null; let outputHealthTimer = null; let assistantLine = null; let assistantBuffer = ''; let inputRecorder = null; let inputRecordingChunks = []; let inputRecordingTrack = null; let inputRecordingStartedAt = 0; let senderRecorder = null; let senderRecordingChunks = []; let senderRecordingTrack = null; let senderRecordingStartedAt = 0; let videoRecorder = null; let videoRecordingChunks = []; let videoRecordingTrack = null; let videoRecordingStartedAt = 0; function log(message) { console.log(`[AI Element2 Gated] ${message}`); } function readBoolSetting(key, fallback) { try { const value = localStorage.getItem(key); if (value === null) return fallback; return value === 'true'; } catch { return fallback; } } function writeBoolSetting(key, value) { try { localStorage.setItem(key, value ? 'true' : 'false'); } catch { // Ignore storage failures; the current checkbox state still applies. } } function readAudioSelectionMode() { try { if (!shouldSendAudioToAi()) return 'off'; return localStorage.getItem(AUDIO_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto'; } catch { return 'auto'; } } function writeAudioSelectionMode(mode) { audioSelectionMode = ['auto', 'manual', 'off'].includes(mode) ? mode : 'auto'; writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, audioSelectionMode !== 'off'); try { localStorage.setItem(AUDIO_SELECTION_MODE_STORAGE_KEY, audioSelectionMode); } catch {} } function readVideoSelectionMode() { try { if (!shouldSendVideoToAi()) return 'off'; return localStorage.getItem(VIDEO_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto'; } catch { return 'auto'; } } function writeVideoSelectionMode(mode) { videoSelectionMode = ['auto', 'manual', 'off'].includes(mode) ? mode : 'auto'; writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, videoSelectionMode !== 'off'); try { localStorage.setItem(VIDEO_SELECTION_MODE_STORAGE_KEY, videoSelectionMode); } catch {} } function readSenderSelectionMode() { try { return localStorage.getItem(SENDER_SELECTION_MODE_STORAGE_KEY) === 'manual' ? 'manual' : 'auto'; } catch { return 'auto'; } } function writeSenderSelectionMode(mode) { senderSelectionMode = mode === 'manual' ? 'manual' : 'auto'; try { localStorage.setItem(SENDER_SELECTION_MODE_STORAGE_KEY, senderSelectionMode); } catch {} } function shouldSendAudioToAi() { return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); } function shouldSendVideoToAi() { return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); } function appendChatLine(role, text, important = false) { const box = document.getElementById('element2-copilot-log'); if (!box) return; if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; const row = document.createElement('div'); row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; row.style.marginBottom = '4px'; if (important) row.style.color = '#fcd34d'; box.appendChild(row); while (box.children.length > 120) box.removeChild(box.firstChild); box.scrollTop = box.scrollHeight; return row; } function startAssistantLine() { assistantBuffer = ''; assistantLine = null; } function appendAssistantDelta(delta) { if (!delta) return; const box = document.getElementById('element2-copilot-log'); if (!box) { console.log(`[AI Element2 Gated] AI: ${delta}`); return; } if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; if (!assistantLine) assistantLine = appendChatLine('AI', ''); assistantBuffer += delta; assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; box.scrollTop = box.scrollHeight; } function finishAssistantLine() { assistantLine = null; assistantBuffer = ''; } function logUserTranscript(text) { if (!text?.trim()) return; finishAssistantLine(); appendChatLine('用户', text.trim(), true); } function recorderMimeType() { if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; return ''; } function recordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `ai-input-${trackPart}-${stamp}.webm`; } function senderRecordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `business-sender-${trackPart}-${stamp}.webm`; } function videoRecordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `ai-video-${trackPart}-${stamp}.webm`; } function videoRecorderMimeType() { if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8'; if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm'; return ''; } async function startInputRecording() { if (inputRecorder?.state === 'recording') return; selectRemoteAudio('record'); if (!isLive(selectedRemoteAudio)) { appendChatLine('录音', `没有可录制的 ${STATIC_AUDIO_ELEMENT_REASON} 音轨`, true); return; } if (!window.MediaRecorder) { appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); return; } inputRecordingChunks = []; inputRecordingTrack = selectedRemoteAudio; inputRecordingStartedAt = Date.now(); const stream = new MediaStream([selectedRemoteAudio]); const mimeType = recorderMimeType(); inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); inputRecorder.ondataavailable = (event) => { if (event.data?.size > 0) inputRecordingChunks.push(event.data); }; inputRecorder.onstop = () => { const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = recordingFileName(inputRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录音', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); inputRecorder = null; inputRecordingChunks = []; inputRecordingTrack = null; updateRecordButton(); }; inputRecorder.start(250); appendChatLine('录音', `开始录制 AI 输入:${trackLabel(inputRecordingTrack)}`, true); updateRecordButton(); } function stopInputRecording() { if (inputRecorder?.state === 'recording') inputRecorder.stop(); } function toggleInputRecording() { if (inputRecorder?.state === 'recording') stopInputRecording(); else startInputRecording(); } function updateRecordButton() { const button = document.getElementById('element2-copilot-record'); if (!button) return; const recording = inputRecorder?.state === 'recording'; button.textContent = recording ? '■' : '●'; button.title = recording ? '停止录制 AI 输入' : '录制 AI 输入'; button.setAttribute('aria-label', button.title); button.style.background = recording ? '#dc2626' : '#2563eb'; } async function startSenderRecording() { if (senderRecorder?.state === 'recording') return; selectBusinessSender('record-sender'); const track = selectedBusinessSender?.track; if (!isLive(track)) { appendChatLine('录音', '当前没有可录制的业务 sender 音轨', true); return; } if (!window.MediaRecorder) { appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); return; } senderRecordingChunks = []; senderRecordingTrack = track; senderRecordingStartedAt = Date.now(); const mimeType = recorderMimeType(); senderRecorder = new MediaRecorder(new MediaStream([track]), mimeType ? { mimeType } : undefined); senderRecorder.ondataavailable = (event) => { if (event.data?.size > 0) senderRecordingChunks.push(event.data); }; senderRecorder.onstop = () => { const blob = new Blob(senderRecordingChunks, { type: senderRecorder?.mimeType || 'audio/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = senderRecordingFileName(senderRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - senderRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录音', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); senderRecorder = null; senderRecordingChunks = []; senderRecordingTrack = null; updateSenderRecordButton(); }; senderRecorder.start(250); appendChatLine('录音', `开始录制业务 sender:${trackLabel(track)}`, true); updateSenderRecordButton(); } function toggleSenderRecording() { if (senderRecorder?.state === 'recording') senderRecorder.stop(); else void startSenderRecording(); } function updateSenderRecordButton() { const button = document.getElementById('element2-copilot-record-sender'); if (!button) return; const recording = senderRecorder?.state === 'recording'; button.textContent = recording ? '■' : '●'; button.title = recording ? '停止录制 sender 音轨' : '录制 sender 音轨'; button.setAttribute('aria-label', button.title); button.style.background = recording ? '#dc2626' : '#7c3aed'; } function updateVideoRecordButton() { const button = document.getElementById('element2-copilot-record-video'); if (!button) return; const recording = videoRecorder?.state === 'recording'; button.textContent = recording ? '■' : '●'; button.title = recording ? '停止录制 AI 视频' : '录制 AI 视频'; button.setAttribute('aria-label', button.title); button.style.background = recording ? '#dc2626' : '#475569'; } async function startVideoRecording() { if (videoRecorder?.state === 'recording') return; selectRemoteVideo('record-video'); if (!isLive(selectedRemoteVideo)) { appendChatLine('录制视频', `没有可录制的 ${STATIC_VIDEO_ELEMENT_REASON} 视频轨`, true); return; } if (!window.MediaRecorder) { appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true); return; } videoRecordingChunks = []; videoRecordingTrack = selectedRemoteVideo; videoRecordingStartedAt = Date.now(); const stream = new MediaStream([videoRecordingTrack]); const mimeType = videoRecorderMimeType(); videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); videoRecorder.ondataavailable = (event) => { if (event.data?.size > 0) videoRecordingChunks.push(event.data); }; videoRecorder.onstop = () => { const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = videoRecordingFileName(videoRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录制视频', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); videoRecorder = null; videoRecordingChunks = []; videoRecordingTrack = null; updateVideoRecordButton(); }; videoRecorder.start(250); appendChatLine('录制视频', `开始录制 AI 视频:${trackLabel(videoRecordingTrack)}`, true); updateVideoRecordButton(); } function stopVideoRecording() { if (videoRecorder?.state === 'recording') videoRecorder.stop(); } function toggleVideoRecording() { if (videoRecorder?.state === 'recording') stopVideoRecording(); else startVideoRecording(); } function pcId() { const bytes = new Uint8Array(16); crypto.getRandomValues(bytes); return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); } function isInternalPc(pc) { return Boolean(pc && (pc.__aiElement2GatedInternal || internalPcs.has(pc) || pc === aiPc)); } function isLive(track) { return track && track.readyState === 'live'; } function trackLabel(track) { if (!track) return 'none'; return `${track.id}${track.muted ? ' muted' : ''}`; } function markInternalPc(pc) { if (!pc) return; pc.__aiElement2GatedInternal = true; internalPcs.add(pc); } function elementReasonFor(element) { if (!element) return null; const tag = element.tagName?.toLowerCase(); if (tag !== 'audio' && tag !== 'video') return null; const elements = Array.from(document.querySelectorAll(tag)); const index = elements.indexOf(element); if (index < 0) return `${tag}-srcObject`; return `element-${index}:${tag}`; } function hasElementReason(item, reason) { return Boolean(item?.reasons?.has(reason)); } function elementAudioReasons(item) { return Array.from(item?.reasons || []).filter((reason) => /^element-\d+:audio$/.test(reason)); } function audioSourceLabel(item) { const reasons = elementAudioReasons(item); return reasons[0] || 'audio-track'; } function shortTrackId(track) { if (!track?.id) return 'none'; return track.id.length <= 22 ? track.id : `${track.id.slice(0, 10)}...${track.id.slice(-7)}`; } function selectableAudioItems() { return Array.from(remoteAudio.values()).filter((item) => ( item?.receiver && isLive(item.track) && !aiOutputTrackIds.has(item.track.id) )); } function audioOptionLabel(item) { const source = audioSourceLabel(item); const state = item.track.muted ? '静音' : '可用'; return `${source} · ${shortTrackId(item.track)} · ${state}`; } function updateAudioTrackSelect() { const select = document.getElementById('element2-copilot-audio-track'); if (!select) return; const items = selectableAudioItems().sort((a, b) => scoreAudioCandidate(b) - scoreAudioCandidate(a)); const desiredValue = audioSelectionMode === 'manual' ? manualAudioTrackId : selectedRemoteAudio?.id; const signature = [ audioSelectionMode, desiredValue || '', ...items.map((item) => `${item.track.id}:${item.track.muted}:${audioSourceLabel(item)}`), ].join('|'); if (signature === audioTrackSelectSignature) return; audioTrackSelectSignature = signature; select.innerHTML = ''; if (items.length === 0) { const option = document.createElement('option'); option.value = ''; option.textContent = '暂无可用远端音轨'; select.appendChild(option); } else { if (audioSelectionMode === 'manual' && !manualAudioTrackId) { const placeholder = document.createElement('option'); placeholder.value = ''; placeholder.textContent = '请选择远端音轨'; select.appendChild(placeholder); } for (const item of items) { const option = document.createElement('option'); option.value = item.track.id; option.textContent = audioOptionLabel(item); select.appendChild(option); } } if (desiredValue && items.some((item) => item.track.id === desiredValue)) select.value = desiredValue; select.disabled = audioSelectionMode !== 'manual'; select.style.opacity = select.disabled ? '0.55' : '1'; } function updateSelectedAudioLabel() { const label = document.getElementById('element2-copilot-selected-audio'); if (!label) return; const mode = audioSelectionMode === 'manual' ? '手动' : audioSelectionMode === 'off' ? '关闭' : '自动'; const item = selectedRemoteAudio ? remoteAudio.get(selectedRemoteAudio.id) : null; label.textContent = selectedRemoteAudio ? `${mode}:${audioSourceLabel(item)} · ${shortTrackId(selectedRemoteAudio)}` : audioSelectionMode === 'off' ? '不传音频给 AI' : `${mode}:尚未选中音轨`; } function pageHasParticipant() { const text = document.body?.innerText || ''; return text.includes('当事人'); } function pageHasOperator() { const text = document.body?.innerText || ''; return text.includes('坐席端'); } function selectedAudioItem() { const item = selectedRemoteAudio ? remoteAudio.get(selectedRemoteAudio.id) : null; return item?.receiver && isLive(item.track) && !item.track.muted ? item : null; } function gateStatus() { const item = selectedAudioItem(); const participantAudioPresent = selectableAudioItems().some((candidate) => !candidate.track.muted); const senderItem = Array.from(businessSenders.values()).find((candidate) => ( candidate.sender === selectedBusinessSender && selectableSenderItems().includes(candidate) )); const participantVisible = pageHasParticipant(); const operatorVisible = pageHasOperator(); return { bothParties: participantVisible && operatorVisible, participantOnline: participantVisible && (Boolean(item) || (audioSelectionMode === 'off' && participantAudioPresent)), operatorOnline: operatorVisible, senderReady: Boolean(senderItem), track: item?.track || null, }; } function gateMessage(status = gateStatus()) { if (!status.operatorOnline) return '等待坐席在线'; if (!status.participantOnline) return '等待当事人在线'; if (!status.senderReady) return '等待通话上行就绪'; return '就绪:可开启 AI 托管'; } function updateGateLabel() { const label = document.getElementById('element2-copilot-gate'); if (!label) return; const status = gateStatus(); label.textContent = gateMessage(status); label.style.color = status.participantOnline && status.operatorOnline && status.senderReady ? '#86efac' : '#fcd34d'; } function scoreAudioCandidate(item) { if (!item || !isLive(item.track) || item.track.muted || aiOutputTrackIds.has(item.track.id)) return -1; let score = 0; if (item.receiver) score += 300; if ((item.bytesReceived || 0) > 0) score += 100; if ((item.bytesDelta || 0) > 0) score += 4000 + Math.min(1500, item.bytesDelta); if ((item.packetsDelta || 0) > 0) score += 2000 + Math.min(800, item.packetsDelta * 8); score += Math.min(1200, (item.audioLevel || 0) * 12000); if (elementAudioReasons(item).length > 0) score += 100; if (hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON)) score += 5000; if (PARTICIPANT_AUDIO_TRACK_PATTERN.test(item.track.id)) score += 5000; item.audioScore = score; return score; } function rememberRemoteReceiver(receiver, reason) { const track = receiver?.track; if (!isLive(track) || isInternalPc(receiver?.transport)) return; if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) { let item = remoteAudio.get(track.id); if (!item) { item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 }; remoteAudio.set(track.id, item); log(`audio candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.receiver = receiver || item.receiver; item.reasons.add(reason); selectRemoteAudio(reason); } if (track.kind === 'video') { let item = remoteVideo.get(track.id); if (!item) { item = { track, receiver, reasons: new Set() }; remoteVideo.set(track.id, item); log(`video candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.receiver = receiver || item.receiver; item.reasons.add(reason); selectRemoteVideo(reason); } } function rememberRemoteStream(stream, reason) { if (!stream || !(stream instanceof MediaStream)) return; for (const track of stream.getAudioTracks()) { if (aiOutputTrackIds.has(track.id)) continue; let item = remoteAudio.get(track.id); if (!item) { item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 }; remoteAudio.set(track.id, item); log(`audio stream candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.reasons.add(reason); } for (const track of stream.getVideoTracks()) { let item = remoteVideo.get(track.id); if (!item) { item = { track, receiver: null, reasons: new Set() }; remoteVideo.set(track.id, item); log(`video stream candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.reasons.add(reason); } selectRemoteAudio(reason); selectRemoteVideo(reason); } function selectRemoteAudio(reason) { const items = selectableAudioItems(); const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); let item = null; if (audioSelectionMode === 'off') { item = null; } else if (audioSelectionMode === 'manual') { item = items.find((candidate) => candidate.track.id === manualAudioTrackId) || null; } else { const ranked = items .filter((candidate) => !candidate.track.muted) .sort((a, b) => scoreAudioCandidate(b) - scoreAudioCandidate(a)); const best = ranked[0] || null; const currentScore = current ? scoreAudioCandidate(current) : -1; const bestScore = best ? scoreAudioCandidate(best) : -1; item = current && currentScore >= 0 && (!best || bestScore < currentScore + 1500) ? current : best; } if (!item) { if (selectedRemoteAudio) { selectedRemoteAudio = null; if (aiAudioSender) void aiAudioSender.replaceTrack(null).catch(() => {}); } updateSelectedAudioLabel(); updateAudioTrackSelect(); updateGateLabel(); return null; } if (selectedRemoteAudio !== item.track) { selectedRemoteAudio = item.track; log(`selected input audio (${reason}) from ${audioSourceLabel(item)}: ${trackLabel(selectedRemoteAudio)}`, true); updateSelectedAudioLabel(); attachAudioToAi(); } updateAudioTrackSelect(); updateGateLabel(); return selectedRemoteAudio; } function selectRemoteVideo(reason) { const items = selectableVideoItems(); let item = null; if (videoSelectionMode === 'manual') { item = items.find((candidate) => candidate.track.id === manualVideoTrackId) || null; } else if (videoSelectionMode === 'auto') { item = items.find((candidate) => hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON)) || items[0] || null; } if (!item) { if (selectedRemoteVideo) { selectedRemoteVideo = null; if (aiVideoSender) void aiVideoSender.replaceTrack(null).catch(() => {}); } updateVideoTrackSelect(); return null; } if (selectedRemoteVideo !== item.track) { selectedRemoteVideo = item.track; log(`selected input video (${reason}): ${trackLabel(selectedRemoteVideo)}`, true); attachVideoToAi(); } updateVideoTrackSelect(); return selectedRemoteVideo; } function selectableVideoItems() { return Array.from(remoteVideo.values()).filter((item) => ( item?.receiver && isLive(item.track) && !item.track.muted )); } function videoSourceLabel(item) { const reason = Array.from(item?.reasons || []).find((value) => /^element-\d+:video$/.test(value)); return reason || 'video-track'; } function updateVideoTrackSelect() { const select = document.getElementById('element2-copilot-video-track'); if (!select) return; const items = selectableVideoItems(); const desiredValue = videoSelectionMode === 'manual' ? manualVideoTrackId : selectedRemoteVideo?.id; const signature = [ videoSelectionMode, desiredValue || '', ...items.map((item) => `${item.track.id}:${videoSourceLabel(item)}`), ].join('|'); if (signature === videoTrackSelectSignature) return; videoTrackSelectSignature = signature; select.innerHTML = ''; if (items.length === 0) { const option = document.createElement('option'); option.value = ''; option.textContent = '暂无可用远端视频'; select.appendChild(option); } else { if (videoSelectionMode === 'manual' && !manualVideoTrackId) { const placeholder = document.createElement('option'); placeholder.value = ''; placeholder.textContent = '请选择远端视频'; select.appendChild(placeholder); } for (const item of items) { const option = document.createElement('option'); option.value = item.track.id; option.textContent = `${videoSourceLabel(item)} · ${shortTrackId(item.track)}`; select.appendChild(option); } } if (desiredValue && items.some((item) => item.track.id === desiredValue)) select.value = desiredValue; select.disabled = videoSelectionMode !== 'manual'; select.style.opacity = select.disabled ? '0.55' : '1'; } function rememberBusinessSender(sender, track, reason) { if (!sender) return; const senderTrack = track || sender.track; if (senderTrack && senderTrack.kind !== 'audio') return; const id = sender.__aiElement2GatedSenderId || `sender-${businessSenders.size + 1}`; sender.__aiElement2GatedSenderId = id; let item = businessSenders.get(id); if (!item) { item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 }; businessSenders.set(id, item); log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true); } item.sender = sender; item.track = sender.track || senderTrack || item.track; if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack; item.sources.add(reason); wrapBusinessSender(sender); selectBusinessSender(reason); } function wrapBusinessSender(sender) { if (!sender || sender.__aiElement2GatedWrapped) return; const originalReplaceTrack = sender.replaceTrack?.bind(sender); if (!originalReplaceTrack) return; sender.replaceTrack = async function (track) { const result = await originalReplaceTrack(track); rememberBusinessSender(sender, track, 'replaceTrack'); return result; }; sender.__aiElement2GatedWrapped = true; } function isRemoteInputTrack(track) { const item = track ? remoteAudio.get(track.id) : null; return Boolean(item?.receiver && !aiOutputTrackIds.has(track.id)); } function senderCandidateTrack(item) { const current = item?.sender?.track; if (current && current !== aiAudioTrack) return current; return item?.originalTrack || item?.track || null; } function selectableSenderItems() { return Array.from(businessSenders.values()).filter((item) => { const track = senderCandidateTrack(item); return isLive(track) && track.kind === 'audio' && !isRemoteInputTrack(track); }); } function scoreSenderCandidate(item) { const track = senderCandidateTrack(item); if (!isLive(track) || isRemoteInputTrack(track)) return -1; let score = 1000; if (!track.muted) score += 400; if (item.sources.has('pc-addTrack')) score += 2000; if ((item.bytesSent || 0) > 0) score += 300; if ((item.bytesDelta || 0) > 0) score += 4500 + Math.min(1800, item.bytesDelta); if ((item.packetsDelta || 0) > 0) score += 1800 + Math.min(900, item.packetsDelta * 10); return score; } function updateSenderSelect() { const select = document.getElementById('element2-copilot-sender'); if (!select) return; const items = selectableSenderItems().sort((a, b) => scoreSenderCandidate(b) - scoreSenderCandidate(a)); const selectedItem = items.find((item) => item.sender === selectedBusinessSender); const desiredValue = senderSelectionMode === 'manual' ? manualSenderId : selectedItem?.id; const signature = [ senderSelectionMode, desiredValue || '', ...items.map((item) => `${item.id}:${senderCandidateTrack(item)?.id || ''}:${senderCandidateTrack(item)?.muted}`), ].join('|'); if (signature === senderSelectSignature) return; senderSelectSignature = signature; select.innerHTML = ''; if (items.length === 0) { const option = document.createElement('option'); option.value = ''; option.textContent = '暂无可用 sender'; select.appendChild(option); } else { if (senderSelectionMode === 'manual' && !manualSenderId) { const placeholder = document.createElement('option'); placeholder.value = ''; placeholder.textContent = '请选择 sender'; select.appendChild(placeholder); } for (const item of items) { const option = document.createElement('option'); option.value = item.id; option.textContent = `${item.id} · ${shortTrackId(senderCandidateTrack(item))}${item.sources.has('pc-addTrack') ? ' · addTrack' : ''}`; select.appendChild(option); } } if (desiredValue && items.some((item) => item.id === desiredValue)) select.value = desiredValue; select.disabled = senderSelectionMode !== 'manual'; select.style.opacity = select.disabled ? '0.55' : '1'; } function selectBusinessSender(reason) { if (changingBusinessSender) return selectedBusinessSender; const items = selectableSenderItems(); const current = items.find((candidate) => candidate.sender === selectedBusinessSender); let item = null; if (senderSelectionMode === 'manual') { item = items.find((candidate) => candidate.id === manualSenderId) || null; } else { item = current || items.sort((a, b) => scoreSenderCandidate(b) - scoreSenderCandidate(a))[0] || null; } if (!item) { if (!aiAttached) { selectedBusinessSender = null; originalBusinessTrack = null; } updateSenderSelect(); return null; } if (selectedBusinessSender === item.sender) { updateSenderSelect(); return selectedBusinessSender; } selectedBusinessSender = item.sender; originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); updateSenderSelect(); updateGateLabel(); attachAiOutputToBusiness(); return selectedBusinessSender; } async function changeBusinessSender(reason) { changingBusinessSender = true; try { await restoreBusinessAudio(); selectedBusinessSender = null; originalBusinessTrack = null; aiAttached = false; } finally { changingBusinessSender = false; } senderSelectSignature = ''; selectBusinessSender(reason); if (isAiTakeover) await attachAiOutputToBusiness(); updateGateLabel(); } function setVu(id, level) { const el = document.getElementById(id); if (!el) return; el.style.width = `${Math.min(100, level * 2500)}%`; el.style.opacity = level > 0 ? '1' : '0.35'; } function volumePercent(level) { if (!Number.isFinite(level) || level < 0.0015) return 0; return Math.round(Math.min(100, level * 2500)); } function volumeDb(level) { if (!Number.isFinite(level) || level < 0.0015) return '-inf'; return `${(20 * Math.log10(level)).toFixed(1)}dB`; } function volumeText(level) { return `vol=${volumePercent(level)}% ${volumeDb(level)}`; } function setVolumeLabel(id, level) { const el = document.getElementById(id); if (!el) return; el.textContent = volumeText(level); } function renderVu() { refreshAiOutputVuFromAnalyser(); inputVu *= 0.88; outputVu *= 0.88; setVu('element2-copilot-input-vu', inputVu); setVu('element2-copilot-output-vu', outputVu); setVolumeLabel('element2-copilot-input-volume', inputVu); setVolumeLabel('element2-copilot-output-volume', outputVu); requestAnimationFrame(renderVu); } function speechVuLevel(level) { const noiseFloor = 0.0015; if (!Number.isFinite(level) || level < noiseFloor) return 0; return Math.min(0.08, (level - noiseFloor) * 3.2); } function analyserVuLevel(analyser, data) { if (!analyser || !data) return 0; analyser.getByteTimeDomainData(data); let sum = 0; for (const value of data) { const centered = (value - 128) / 128; sum += centered * centered; } return speechVuLevel(Math.sqrt(sum / data.length)); } function refreshAiOutputVuFromAnalyser() { outputVu = Math.max(outputVu, analyserVuLevel(aiOutputAnalyser, aiOutputAnalyserData)); } function createAudioContext() { const AudioContextCtor = window.AudioContext || window.webkitAudioContext; return AudioContextCtor ? new AudioContextCtor() : null; } function detachAiOutputMeter() { try { aiOutputAnalyserSource?.disconnect(); } catch {} aiOutputAnalyser = null; aiOutputAnalyserData = null; aiOutputAnalyserSource = null; aiOutputAnalyserTrack = null; outputVu = 0; } async function attachAiOutputMeter(track) { if (!isLive(track)) { detachAiOutputMeter(); return false; } if (aiOutputAnalyserTrack === track && aiOutputAnalyser) return true; detachAiOutputMeter(); if (!aiOutputAudioContext) { aiOutputAudioContext = createAudioContext(); if (!aiOutputAudioContext) { appendChatLine('音频', '当前浏览器不支持 AudioContext,无法显示 AI 输出音量', true); return false; } } if (aiOutputAudioContext.state === 'suspended') { await aiOutputAudioContext.resume().catch(() => {}); } try { const stream = new MediaStream([track]); aiOutputAnalyserSource = aiOutputAudioContext.createMediaStreamSource(stream); aiOutputAnalyser = aiOutputAudioContext.createAnalyser(); aiOutputAnalyser.fftSize = 512; aiOutputAnalyser.smoothingTimeConstant = 0.55; aiOutputAnalyserData = new Uint8Array(aiOutputAnalyser.fftSize); aiOutputAnalyserSource.connect(aiOutputAnalyser); aiOutputAnalyserTrack = track; return true; } catch (error) { log(`failed to attach AI output meter: ${error.message || error}`); detachAiOutputMeter(); return false; } } async function refreshStats() { await refreshInputStats(); await refreshOutputStats(); } async function refreshInputStats() { for (const item of remoteAudio.values()) { if (!item.receiver?.getStats) continue; const report = await item.receiver.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; const prev = inboundMemory.get(stat.id); const bytes = stat.bytesReceived || 0; const packets = stat.packetsReceived || 0; let audioLevel = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0; if ( prev && typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number' && typeof prev.energy === 'number' && typeof prev.duration === 'number' ) { const energyDelta = stat.totalAudioEnergy - prev.energy; const durationDelta = stat.totalSamplesDuration - prev.duration; if (energyDelta > 0 && durationDelta > 0) { audioLevel = Math.max(audioLevel, Math.sqrt(energyDelta / durationDelta)); } } item.bytesReceived = bytes; item.packetsReceived = packets; item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; item.audioLevel = audioLevel; inboundMemory.set(stat.id, { bytes, packets, energy: stat.totalAudioEnergy, duration: stat.totalSamplesDuration, }); if (item.track === selectedRemoteAudio) { inputVu = Math.max(inputVu, speechVuLevel(audioLevel)); } }); } selectRemoteAudio('stats'); } async function refreshOutputStats() { for (const item of businessSenders.values()) { if (!item.sender?.getStats) continue; item.track = item.sender.track || item.track; const report = await item.sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; const key = `${item.id}:${stat.id}`; const prev = outboundMemory.get(key); const bytes = stat.bytesSent || 0; const packets = stat.packetsSent || 0; item.bytesSent = bytes; item.packetsSent = packets; item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; outboundMemory.set(key, { bytes, packets }); }); } selectBusinessSender('stats'); updateSenderSelect(); } async function outboundBytes(sender) { if (!sender?.getStats) return null; let total = 0; const report = await sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0; }); return total; } function aiState() { if (!aiPc) return 'pc=null'; return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; } function aiReady() { return Boolean(aiPc?.remoteDescription && ( aiPc.connectionState === 'connected' || aiPc.iceConnectionState === 'connected' || aiPc.iceConnectionState === 'completed' )); } function waitForAiReady(timeoutMs = 10000) { if (aiReady()) return Promise.resolve(true); return new Promise((resolve) => { const startedAt = Date.now(); const timer = setInterval(() => { if (aiReady()) { clearInterval(timer); resolve(true); } else if (Date.now() - startedAt >= timeoutMs) { clearInterval(timer); resolve(false); } }, 250); }); } function checkAiInput(track) { if (inputHealthTimer) clearTimeout(inputHealthTimer); inputHealthTimer = setTimeout(async () => { if (!aiAudioSender || aiAudioSender.track !== track) return; await waitForAiReady(); const before = await outboundBytes(aiAudioSender); await new Promise((resolve) => setTimeout(resolve, 2500)); const after = await outboundBytes(aiAudioSender); log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true); }, 500); } function checkAiOutput(track) { if (outputHealthTimer) clearTimeout(outputHealthTimer); outputHealthTimer = setTimeout(async () => { if (!selectedBusinessSender || selectedBusinessSender.track !== track) return; const before = await outboundBytes(selectedBusinessSender); await new Promise((resolve) => setTimeout(resolve, 2500)); const after = await outboundBytes(selectedBusinessSender); log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true); }, 800); } async function attachAudioToAi() { if (!shouldSendAudioToAi()) return false; if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false; if (aiAudioSender.track === selectedRemoteAudio) return true; await aiAudioSender.replaceTrack(selectedRemoteAudio); log(`AI input uses ${trackLabel(selectedRemoteAudio)} from ${STATIC_AUDIO_ELEMENT_REASON}`, true); checkAiInput(selectedRemoteAudio); return true; } async function attachVideoToAi() { if (!shouldSendVideoToAi()) return false; if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false; if (aiVideoSender.track === selectedRemoteVideo) return true; await aiVideoSender.replaceTrack(selectedRemoteVideo); log(`AI video uses ${trackLabel(selectedRemoteVideo)} from ${STATIC_VIDEO_ELEMENT_REASON}`, true); return true; } async function attachAiOutputToBusiness() { selectBusinessSender('before-output'); if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) { log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`); return false; } if (selectedBusinessSender.track === aiAudioTrack) return true; await selectedBusinessSender.replaceTrack(aiAudioTrack); aiAttached = true; log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true); checkAiOutput(aiAudioTrack); return true; } async function restoreBusinessAudio() { if (!selectedBusinessSender || !aiAttached) return; await selectedBusinessSender.replaceTrack(originalBusinessTrack || null); aiAttached = false; log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true); } function wrapTrackListener(listener) { if (!listener || listener.__aiElement2GatedWrapped) return listener; const wrapped = function (event) { const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget; if (!isInternalPc(ownerPc) && event?.track) { rememberRemoteReceiver(event.receiver, 'pc-track'); for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); } if (typeof listener === 'function') return listener.apply(this, arguments); if (listener?.handleEvent) return listener.handleEvent(event); return undefined; }; wrapped.__aiElement2GatedWrapped = true; return wrapped; } RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); return originalAddEventListener.call(this, type, listener, options); }; try { const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { configurable: true, enumerable: true, get() { return descriptor?.get ? descriptor.get.call(this) : this.__aiElement2GatedOnTrack || null; }, set(listener) { const wrapped = wrapTrackListener(listener); this.__aiElement2GatedOnTrack = wrapped; if (descriptor?.set) descriptor.set.call(this, wrapped); else this.addEventListener('track', wrapped); }, }); } catch (error) { console.warn('[AI Element2 Gated] failed to wrap ontrack', error); } RTCPeerConnection.prototype.addTrack = function (track, ...streams) { const sender = originalAddTrack.call(this, track, ...streams); if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack'); return sender; }; if (originalAddTransceiver) { RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { const transceiver = originalAddTransceiver.call(this, trackOrKind, init); if (!isInternalPc(this)) { const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; if (kind === 'audio') { const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver'); } } return transceiver; }; } RTCPeerConnection.prototype.setRemoteDescription = function () { const result = originalSetRemoteDescription.apply(this, arguments); if (!isInternalPc(this)) { Promise.resolve(result).then(() => { for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd'); for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd'); }).catch(() => {}); } return result; }; if (mediaSrcObjectDescriptor?.set) { Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { configurable: true, enumerable: mediaSrcObjectDescriptor.enumerable, get() { return mediaSrcObjectDescriptor.get.call(this); }, set(stream) { if (!this.__aiElement2GatedCopilotElement) { const reason = elementReasonFor(this) || `${this.tagName.toLowerCase()}-srcObject`; rememberRemoteStream(stream, reason); } return mediaSrcObjectDescriptor.set.call(this, stream); }, }); } function scanMediaElements() { document.querySelectorAll('audio').forEach((element, index) => { if (element.__aiElement2GatedCopilotElement) return; rememberRemoteStream(element.srcObject, `element-${index}:audio`); }); document.querySelectorAll('video').forEach((element, index) => { if (element.__aiElement2GatedCopilotElement) return; rememberRemoteStream(element.srcObject, `element-${index}:video`); }); selectRemoteAudio('media-scan'); selectRemoteVideo('media-scan'); updateGateLabel(); } setInterval(scanMediaElements, 1000); function endpointUrl(baseUrl, path) { const normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, ''); return new URL(`${normalizedBaseUrl}${path}`).toString(); } function basicAuthorization(username, password) { const bytes = new TextEncoder().encode(`${username}:${password}`); let binary = ''; bytes.forEach((byte) => { binary += String.fromCharCode(byte); }); return `Basic ${btoa(binary)}`; } function aiRequestHeaders(extra = {}) { return { ...extra, ...(aiAuthorization ? { Authorization: aiAuthorization } : {}), }; } async function fetchIceServers(baseUrl) { try { const iceUrl = endpointUrl(baseUrl, '/api/webrtc/ice-servers'); const response = await fetch(iceUrl, { credentials: 'include', headers: aiRequestHeaders(), }); if (!response.ok) throw new Error(`HTTP ${response.status}`); const payload = await response.json(); if (Array.isArray(payload.iceServers)) return payload.iceServers; } catch (error) { log(`ICE server config unavailable, using public STUN: ${error.message || error}`); } return [{ urls: 'stun:stun.l.google.com:19302' }]; } function queueAiIceCandidate(candidate) { if (!candidate) return; aiCandidateQueue.push({ candidate: candidate.candidate, sdp_mid: candidate.sdpMid, sdp_mline_index: candidate.sdpMLineIndex, }); if (!aiCandidateFlushTimer) { aiCandidateFlushTimer = setTimeout(flushAiIceCandidates, 200); } } async function flushAiIceCandidates() { aiCandidateFlushTimer = null; if (!aiOfferUrl || !aiPcId || !aiCanSendCandidates || aiCandidateQueue.length === 0) return; const candidates = aiCandidateQueue.splice(0, aiCandidateQueue.length); try { const response = await fetch(aiOfferUrl, { method: 'PATCH', credentials: 'include', headers: aiRequestHeaders({ 'Content-Type': 'application/json' }), body: JSON.stringify({ pc_id: aiPcId, candidates }), }); if (!response.ok) throw new Error(`HTTP ${response.status}`); } catch (error) { aiCandidateQueue.unshift(...candidates); log(`failed to send ICE candidates: ${error.message || error}`); } } function sendAiSignalling(message) { if (aiDc?.readyState !== 'open') return; aiDc.send(JSON.stringify({ type: 'signalling', message })); } async function negotiateAi(restartPc = false) { if (!aiPc || !aiOfferUrl || !aiAssistantId || aiNegotiating) return false; aiNegotiating = true; try { const offer = await aiPc.createOffer(); await aiPc.setLocalDescription(offer); const response = await fetch(aiOfferUrl, { method: 'POST', credentials: 'include', headers: aiRequestHeaders({ 'Content-Type': 'application/json', 'X-Pipecat-Assistant-ID': aiAssistantId, }), body: JSON.stringify({ pc_id: aiPcId, sdp: aiPc.localDescription.sdp, type: aiPc.localDescription.type, restart_pc: restartPc, requestData: { assistant_id: aiAssistantId, vision_enabled: shouldSendVideoToAi(), dynamic_variables: {}, }, }), }); if (!response.ok) { const detail = await response.text().catch(() => ''); throw new Error(`HTTP ${response.status}${detail ? `: ${detail}` : ''}`); } const answer = await response.json(); aiPcId = answer.pc_id; await aiPc.setRemoteDescription({ type: answer.type, sdp: answer.sdp }); aiCanSendCandidates = true; await flushAiIceCandidates(); log(`AI WebRTC answer applied; ${aiState()}`, true); return true; } finally { aiNegotiating = false; } } async function handleAiSignalling(message) { try { if (message?.type === 'renegotiate') { await negotiateAi(false); } else if (message?.type === 'peerLeft') { setStatus('disconnected'); log('AI peer left', true); } else { log(`unknown SmallWebRTC signalling: ${message?.type || 'missing type'}`); } } catch (error) { log(`SmallWebRTC renegotiation failed: ${error.message || error}`, true); } } async function connectAi(baseUrl, assistantId, username, password) { scanMediaElements(); selectRemoteAudio('connect'); selectRemoteVideo('connect'); selectBusinessSender('connect'); const status = gateStatus(); updateGateLabel(); if (!status.operatorOnline || !status.participantOnline || !status.senderReady) { appendChatLine('门槛', gateMessage(status), true); setStatus('disconnected'); return false; } disconnectAi(); selectedRemoteAudio = status.track; try { aiOfferUrl = endpointUrl(baseUrl, '/api/webrtc/offer'); } catch { appendChatLine('配置', 'Base URL 格式无效,请填写包含 http:// 或 https:// 的完整地址', true); setStatus('failed'); return false; } aiAssistantId = assistantId; aiAuthorization = basicAuthorization(username, password); aiPcId = pcId(); aiCandidateQueue = []; aiCanSendCandidates = false; const iceServers = await fetchIceServers(baseUrl); aiPc = new RTCPeerConnection({ iceServers }); markInternalPc(aiPc); aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); aiDc = aiPc.createDataChannel('chat'); aiDc.onopen = () => { setStatus('connected'); log('AI data channel open', true); sendAiSignalling({ type: 'trackStatus', receiver_index: 0, enabled: shouldSendAudioToAi() }); sendAiSignalling({ type: 'trackStatus', receiver_index: 1, enabled: shouldSendVideoToAi() }); aiDc.send(JSON.stringify({ type: 'client-ready' })); aiKeepAliveTimer = setInterval(() => { if (aiDc?.readyState === 'open') aiDc.send(`ping: ${Date.now()}`); }, 1000); }; aiDc.onclose = () => setStatus('disconnected'); aiDc.onmessage = (event) => { try { const msg = JSON.parse(event.data); const content = msg.delta || msg.content; if (msg.type === 'signalling') { void handleAiSignalling(msg.message); } else if (msg.type === 'transcript') { logUserTranscript(content); } else if (msg.type === 'assistant-text-start') { startAssistantLine(); } else if (msg.type === 'assistant-text-delta') { appendAssistantDelta(content); } else if (msg.type === 'assistant-text-end') { finishAssistantLine(); } else { log(`data channel message: ${msg.type}`); } } catch { appendChatLine('DATA', event.data); } }; const sendAudio = shouldSendAudioToAi(); const audioTransceiver = aiPc.addTransceiver('audio', { direction: 'sendrecv' }); aiAudioSender = audioTransceiver.sender; if (sendAudio && isLive(selectedRemoteAudio)) await aiAudioSender.replaceTrack(selectedRemoteAudio); log(!sendAudio ? 'AI audio uplink disabled by panel setting' : isLive(selectedRemoteAudio) ? `AI input starts with ${trackLabel(selectedRemoteAudio)} (${STATIC_AUDIO_ELEMENT_REASON})` : `AI input waiting for ${STATIC_AUDIO_ELEMENT_REASON}`, true); const sendVideo = shouldSendVideoToAi(); const shouldSendVideo = sendVideo && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; const videoTransceiver = aiPc.addTransceiver('video', { direction: 'sendrecv' }); aiVideoSender = videoTransceiver.sender; if (shouldSendVideo) { await aiVideoSender.replaceTrack(selectedRemoteVideo); log(`AI video starts with ${trackLabel(selectedRemoteVideo)} (${STATIC_VIDEO_ELEMENT_REASON})`); } else { log(sendVideo ? `AI video disabled: waiting for live track at ${STATIC_VIDEO_ELEMENT_REASON}` : 'AI video uplink disabled by panel setting'); } aiPc.ontrack = (event) => { if (event.track.kind !== 'audio') return; aiOutputTrackIds.add(event.track.id); remoteAudio.delete(event.track.id); aiAudioTrack = event.track; attachAiOutputMeter(event.track); log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); aiAudioTrack.onunmute = () => { log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); attachAiOutputToBusiness(); }; attachAiOutputToBusiness(); }; aiPc.onicecandidate = (event) => { queueAiIceCandidate(event.candidate); }; setStatus('connecting'); try { await negotiateAi(false); log('sent SmallWebRTC HTTP offer', true); } catch (error) { log(`AI HTTP signalling error: ${error.message || error}`, true); disconnectAi(); setStatus('failed'); return false; } attachAudioToAi(); attachVideoToAi(); return true; } function disconnectAi() { try { aiDc?.close(); } catch {} try { aiPc?.close(); } catch {} if (aiCandidateFlushTimer) clearTimeout(aiCandidateFlushTimer); if (aiKeepAliveTimer) clearInterval(aiKeepAliveTimer); aiDc = null; aiPc = null; aiPcId = null; aiOfferUrl = null; aiAssistantId = null; aiAuthorization = null; aiCandidateQueue = []; aiCandidateFlushTimer = null; aiKeepAliveTimer = null; aiNegotiating = false; aiCanSendCandidates = false; aiAudioSender = null; aiVideoSender = null; aiAudioTrack = null; detachAiOutputMeter(); if (inputRecorder?.state === 'recording') stopInputRecording(); if (senderRecorder?.state === 'recording') senderRecorder.stop(); if (videoRecorder?.state === 'recording') stopVideoRecording(); setStatus('disconnected'); } function setStatus(status) { const el = document.getElementById('element2-copilot-status'); if (!el) return; el.textContent = { disconnected: '未连接', connecting: '连接中', connected: '已连接', failed: '连接失败', }[status] || status; } function updateModeButton() { const button = document.getElementById('element2-copilot-toggle'); if (!button) return; if (isAiTakeover) { button.textContent = '当前:AI 接管|点击切回人工'; button.style.background = '#8b5cf6'; } else { button.textContent = '当前:人工接管|点击切换 AI'; button.style.background = '#10b981'; } } window.__aiElement2GatedDump = function () { const status = gateStatus(); return { audioSelection: { mode: audioSelectionMode, manualTrackId: manualAudioTrackId, preferredElement: STATIC_AUDIO_ELEMENT_REASON, }, videoSelection: { mode: videoSelectionMode, manualTrackId: manualVideoTrackId, preferredElement: STATIC_VIDEO_ELEMENT_REASON, }, senderSelection: { mode: senderSelectionMode, manualSenderId, }, staticVideoElement: STATIC_VIDEO_ELEMENT_REASON, gate: { bothParties: status.bothParties, participantOnline: status.participantOnline, operatorOnline: status.operatorOnline, senderReady: status.senderReady, track: trackLabel(status.track), }, gateMessage: gateMessage(status), selectedInput: trackLabel(selectedRemoteAudio), selectedVideo: trackLabel(selectedRemoteVideo), aiVideoSenderTrack: trackLabel(aiVideoSender?.track), selectedOutput: trackLabel(selectedBusinessSender?.track), originalOutput: trackLabel(originalBusinessTrack), aiAudio: trackLabel(aiAudioTrack), aiAttached, aiState: aiState(), recording: inputRecorder?.state === 'recording' ? { track: trackLabel(inputRecordingTrack), seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), chunks: inputRecordingChunks.length, } : null, senderRecording: senderRecorder?.state === 'recording' ? { track: trackLabel(senderRecordingTrack), seconds: ((Date.now() - senderRecordingStartedAt) / 1000).toFixed(1), chunks: senderRecordingChunks.length, } : null, videoRecording: videoRecorder?.state === 'recording' ? { track: trackLabel(videoRecordingTrack), seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1), chunks: videoRecordingChunks.length, } : null, inputs: Array.from(remoteAudio.values()).map((item) => ({ selected: item.track === selectedRemoteAudio, id: item.track.id, muted: item.track.muted, live: item.track.readyState, hasReceiver: Boolean(item.receiver), source: audioSourceLabel(item), bytesReceived: item.bytesReceived || 0, packetsReceived: item.packetsReceived || 0, bytesDelta: item.bytesDelta, packetsDelta: item.packetsDelta, audioLevel: item.audioLevel || 0, audioScore: Math.round(item.audioScore || scoreAudioCandidate(item) || 0), matchesStaticElement: hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON), reasons: Array.from(item.reasons), })), videos: Array.from(remoteVideo.values()).map((item) => ({ selected: item.track === selectedRemoteVideo, id: item.track.id, muted: item.track.muted, live: item.track.readyState, hasReceiver: Boolean(item.receiver), matchesStaticElement: hasElementReason(item, STATIC_VIDEO_ELEMENT_REASON), reasons: Array.from(item.reasons), })), outputs: Array.from(businessSenders.values()).map((item) => ({ selected: item.sender === selectedBusinessSender, id: item.id, track: trackLabel(item.sender.track || item.track), originalTrack: trackLabel(item.originalTrack), pcAddTrack: item.sources.has('pc-addTrack'), isRemoteInput: isRemoteInputTrack(item.sender.track || item.track), bytesSent: item.bytesSent, packetsSent: item.packetsSent, bytesDelta: item.bytesDelta, packetsDelta: item.packetsDelta, sources: Array.from(item.sources), })), }; }; function renderPanel() { if (!document.body || document.getElementById('element2-copilot-panel')) return; const panel = document.createElement('div'); panel.id = 'element2-copilot-panel'; panel.style.cssText = [ 'position:fixed', 'top:20px', 'left:20px', 'width:440px', 'max-height:calc(100vh - 40px)', 'overflow:auto', 'z-index:999999', 'box-sizing:border-box', 'padding:16px', 'border-radius:10px', 'background:rgba(17,24,39,.96)', 'color:white', 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', 'box-shadow:0 12px 32px rgba(0,0,0,.35)', ].join(';'); panel.innerHTML = `
AI 托管 未连接
等待当事人在线
截取给 AI 的音频音量vol=0% -inf
AI 发送给当事人的音频音量vol=0% -inf
等待交互...
`; document.body.appendChild(panel); const toggle = document.getElementById('element2-copilot-toggle'); const recordButton = document.getElementById('element2-copilot-record'); const senderRecordButton = document.getElementById('element2-copilot-record-sender'); const videoRecordButton = document.getElementById('element2-copilot-record-video'); const urlInput = document.getElementById('element2-copilot-url'); const assistantInput = document.getElementById('element2-copilot-assistant'); const usernameInput = document.getElementById('element2-copilot-username'); const passwordInput = document.getElementById('element2-copilot-password'); const audioModeInput = document.getElementById('element2-copilot-audio-mode'); const audioTrackInput = document.getElementById('element2-copilot-audio-track'); const videoModeInput = document.getElementById('element2-copilot-video-mode'); const videoTrackInput = document.getElementById('element2-copilot-video-track'); const senderModeInput = document.getElementById('element2-copilot-sender-mode'); const senderInput = document.getElementById('element2-copilot-sender'); audioModeInput.value = audioSelectionMode; audioModeInput.addEventListener('change', () => { writeAudioSelectionMode(audioModeInput.value); if (audioSelectionMode === 'manual') { manualAudioTrackId = selectedRemoteAudio?.id || selectableAudioItems()[0]?.track.id || null; } selectRemoteAudio('mode-change'); log(`AI input selection mode: ${audioSelectionMode}`); }); audioTrackInput.addEventListener('change', () => { manualAudioTrackId = audioTrackInput.value || null; selectRemoteAudio('manual-select'); log(`manual AI input track: ${manualAudioTrackId || 'none'}`); }); videoModeInput.value = videoSelectionMode; videoModeInput.addEventListener('change', () => { writeVideoSelectionMode(videoModeInput.value); if (videoSelectionMode === 'manual') { manualVideoTrackId = selectedRemoteVideo?.id || selectableVideoItems()[0]?.track.id || null; } videoTrackSelectSignature = ''; selectRemoteVideo('video-mode-change'); log(`AI video selection mode: ${videoSelectionMode}`); }); videoTrackInput.addEventListener('change', () => { manualVideoTrackId = videoTrackInput.value || null; selectRemoteVideo('manual-video-select'); log(`manual AI video track: ${manualVideoTrackId || 'none'}`); }); senderModeInput.value = senderSelectionMode; senderModeInput.addEventListener('change', async () => { writeSenderSelectionMode(senderModeInput.value); if (senderSelectionMode === 'manual') { const selectedItem = Array.from(businessSenders.values()).find((item) => item.sender === selectedBusinessSender); manualSenderId = selectedItem?.id || selectableSenderItems()[0]?.id || null; } await changeBusinessSender('sender-mode-change'); log(`business sender selection mode: ${senderSelectionMode}`); }); senderInput.addEventListener('change', async () => { manualSenderId = senderInput.value || null; await changeBusinessSender('manual-sender-select'); log(`manual business sender: ${manualSenderId || 'none'}`); }); recordButton.addEventListener('click', toggleInputRecording); senderRecordButton.addEventListener('click', toggleSenderRecording); videoRecordButton.addEventListener('click', toggleVideoRecording); toggle.addEventListener('click', async () => { if (!isAiTakeover && !assistantInput.value.trim()) { alert('请先填写助手 ID。'); return; } if (!isAiTakeover && (!usernameInput.value.trim() || !passwordInput.value)) { alert('请填写管理员用户名和密码。'); return; } isAiTakeover = !isAiTakeover; updateModeButton(); if (isAiTakeover) { const connected = await connectAi( urlInput.value.trim(), assistantInput.value.trim(), usernameInput.value.trim(), passwordInput.value, ); if (connected) { await attachAiOutputToBusiness(); } else { isAiTakeover = false; updateModeButton(); } } else { await restoreBusinessAudio(); disconnectAi(); } }); updateModeButton(); updateRecordButton(); updateSenderRecordButton(); updateVideoRecordButton(); selectRemoteAudio('panel-ready'); selectRemoteVideo('panel-ready'); selectBusinessSender('panel-ready'); updateGateLabel(); } function renderPanelSoon() { try { renderPanel(); } catch (error) { console.warn('[AI Element2 Gated] panel error', error); } } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true }); } else { renderPanelSoon(); } requestAnimationFrame(renderVu); setInterval(refreshStats, 250); setTimeout(renderPanelSoon, 1500); setTimeout(renderPanelSoon, 3000); window.addEventListener('beforeunload', () => { if (inputRecorder?.state === 'recording') stopInputRecording(); if (senderRecorder?.state === 'recording') senderRecorder.stop(); if (videoRecorder?.state === 'recording') stopVideoRecording(); }); })();