// ==UserScript== // @name 12345 AI Element2 Gated Copilot // @namespace http://tampermonkey.net/ // @version 1.0 // @description Two-party gated bridge: only feed element-2:audio into AI after both sides are connected. // @author You // @match *://yhy-yw.22sj.cn:*/* // @grant none // @run-at document-start // ==/UserScript== (function () { 'use strict'; const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio'; const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video'; const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; const originalAddTrack = RTCPeerConnection.prototype.addTrack; const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); const remoteAudio = new Map(); const remoteVideo = new Map(); const businessSenders = new Map(); const aiOutputTrackIds = new Set(); const inboundMemory = new Map(); const outboundMemory = new Map(); const internalPcs = new WeakSet(); const AUDIO_UPLINK_STORAGE_KEY = 'element2-copilot-send-audio'; const VIDEO_UPLINK_STORAGE_KEY = 'element2-copilot-send-video'; let selectedRemoteAudio = null; let selectedRemoteVideo = null; let selectedBusinessSender = null; let originalBusinessTrack = null; let aiAudioTrack = null; let aiPc = null; let aiWs = null; let aiDc = null; let aiAudioSender = null; let aiVideoSender = null; let isAiTakeover = false; let aiAttached = false; let inputVu = 0; let outputVu = 0; let inputHealthTimer = null; let outputHealthTimer = null; let assistantLine = null; let assistantBuffer = ''; let inputRecorder = null; let inputRecordingChunks = []; let inputRecordingTrack = null; let inputRecordingStartedAt = 0; let videoRecorder = null; let videoRecordingChunks = []; let videoRecordingTrack = null; let videoRecordingStartedAt = 0; function log(message) { console.log(`[AI Element2 Gated] ${message}`); } function readBoolSetting(key, fallback) { try { const value = localStorage.getItem(key); if (value === null) return fallback; return value === 'true'; } catch { return fallback; } } function writeBoolSetting(key, value) { try { localStorage.setItem(key, value ? 'true' : 'false'); } catch { // Ignore storage failures; the current checkbox state still applies. } } function shouldSendAudioToAi() { return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); } function shouldSendVideoToAi() { return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); } function appendChatLine(role, text, important = false) { const box = document.getElementById('element2-copilot-log'); if (!box) return; if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; const row = document.createElement('div'); row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; row.style.marginBottom = '4px'; if (important) row.style.color = '#fcd34d'; box.appendChild(row); while (box.children.length > 120) box.removeChild(box.firstChild); box.scrollTop = box.scrollHeight; return row; } function startAssistantLine() { assistantBuffer = ''; assistantLine = null; } function appendAssistantDelta(delta) { if (!delta) return; const box = document.getElementById('element2-copilot-log'); if (!box) { console.log(`[AI Element2 Gated] AI: ${delta}`); return; } if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; if (!assistantLine) assistantLine = appendChatLine('AI', ''); assistantBuffer += delta; assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; box.scrollTop = box.scrollHeight; } function finishAssistantLine() { assistantLine = null; assistantBuffer = ''; } function logUserTranscript(text) { if (!text?.trim()) return; finishAssistantLine(); appendChatLine('用户', text.trim(), true); } function recorderMimeType() { if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; return ''; } function recordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `ai-input-${trackPart}-${stamp}.webm`; } function videoRecordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `ai-video-${trackPart}-${stamp}.webm`; } function videoRecorderMimeType() { if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8'; if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm'; return ''; } async function startInputRecording() { if (inputRecorder?.state === 'recording') return; selectRemoteAudio('record'); if (!isLive(selectedRemoteAudio)) { appendChatLine('录音', `没有可录制的 ${STATIC_AUDIO_ELEMENT_REASON} 音轨`, true); return; } if (!window.MediaRecorder) { appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); return; } inputRecordingChunks = []; inputRecordingTrack = selectedRemoteAudio; inputRecordingStartedAt = Date.now(); const stream = new MediaStream([selectedRemoteAudio]); const mimeType = recorderMimeType(); inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); inputRecorder.ondataavailable = (event) => { if (event.data?.size > 0) inputRecordingChunks.push(event.data); }; inputRecorder.onstop = () => { const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = recordingFileName(inputRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录音', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); inputRecorder = null; inputRecordingChunks = []; inputRecordingTrack = null; updateRecordButton(); }; inputRecorder.start(250); appendChatLine('录音', `开始录制 AI 输入:${trackLabel(inputRecordingTrack)}`, true); updateRecordButton(); } function stopInputRecording() { if (inputRecorder?.state === 'recording') inputRecorder.stop(); } function toggleInputRecording() { if (inputRecorder?.state === 'recording') stopInputRecording(); else startInputRecording(); } function updateRecordButton() { const button = document.getElementById('element2-copilot-record'); if (!button) return; const recording = inputRecorder?.state === 'recording'; button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; button.style.background = recording ? '#dc2626' : '#2563eb'; } function updateVideoRecordButton() { const button = document.getElementById('element2-copilot-record-video'); if (!button) return; const recording = videoRecorder?.state === 'recording'; button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频'; button.style.background = recording ? '#dc2626' : '#475569'; } async function startVideoRecording() { if (videoRecorder?.state === 'recording') return; selectRemoteVideo('record-video'); if (!isLive(selectedRemoteVideo)) { appendChatLine('录制视频', `没有可录制的 ${STATIC_VIDEO_ELEMENT_REASON} 视频轨`, true); return; } if (!window.MediaRecorder) { appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true); return; } videoRecordingChunks = []; videoRecordingTrack = selectedRemoteVideo; videoRecordingStartedAt = Date.now(); const stream = new MediaStream([videoRecordingTrack]); const mimeType = videoRecorderMimeType(); videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); videoRecorder.ondataavailable = (event) => { if (event.data?.size > 0) videoRecordingChunks.push(event.data); }; videoRecorder.onstop = () => { const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = videoRecordingFileName(videoRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录制视频', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); videoRecorder = null; videoRecordingChunks = []; videoRecordingTrack = null; updateVideoRecordButton(); }; videoRecorder.start(250); appendChatLine('录制视频', `开始录制 AI 视频:${trackLabel(videoRecordingTrack)}`, true); updateVideoRecordButton(); } function stopVideoRecording() { if (videoRecorder?.state === 'recording') videoRecorder.stop(); } function toggleVideoRecording() { if (videoRecorder?.state === 'recording') stopVideoRecording(); else startVideoRecording(); } function pcId() { const bytes = new Uint8Array(16); crypto.getRandomValues(bytes); return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); } function isInternalPc(pc) { return Boolean(pc && (pc.__aiElement2GatedInternal || internalPcs.has(pc) || pc === aiPc)); } function isLive(track) { return track && track.readyState === 'live'; } function trackLabel(track) { if (!track) return 'none'; return `${track.id}${track.muted ? ' muted' : ''}`; } function markInternalPc(pc) { if (!pc) return; pc.__aiElement2GatedInternal = true; internalPcs.add(pc); } function elementReasonFor(element) { if (!element) return null; const tag = element.tagName?.toLowerCase(); if (tag !== 'audio' && tag !== 'video') return null; const elements = Array.from(document.querySelectorAll(tag)); const index = elements.indexOf(element); if (index < 0) return `${tag}-srcObject`; return `element-${index}:${tag}`; } function hasElementReason(item, reason) { return Boolean(item?.reasons?.has(reason)); } function elementAudioReasons(item) { return Array.from(item?.reasons || []).filter((reason) => /^element-\d+:audio$/.test(reason)); } function audioSourceLabel(item) { const reasons = elementAudioReasons(item); return reasons[0] || 'audio-track'; } function updateSelectedAudioLabel() { const label = document.getElementById('element2-copilot-selected-audio'); if (!label) return; const item = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); label.textContent = item ? `${audioSourceLabel(item)} ${selectedRemoteAudio.id.slice(0, 8)} score=${Math.round(item.audioScore || 0)}` : `等待 ${STATIC_AUDIO_ELEMENT_REASON}`; } function pageHasBothParties() { const text = document.body?.innerText || ''; return text.includes('当事人') && text.includes('坐席端'); } function staticAudioItem() { return Array.from(remoteAudio.values()).find((candidate) => ( isLive(candidate.track) && !candidate.track.muted && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON) )) || null; } function gateStatus() { const item = staticAudioItem(); return { bothParties: pageHasBothParties(), audioReady: Boolean(item), senderReady: Boolean(selectedBusinessSender), track: item?.track || null, }; } function gateMessage(status = gateStatus()) { if (!status.bothParties) return '等待当事人和坐席端同时在线'; if (!status.audioReady) return `等待 ${STATIC_AUDIO_ELEMENT_REASON} 有声轨`; if (!status.senderReady) return '等待业务上行 sender'; return `就绪:${STATIC_AUDIO_ELEMENT_REASON} -> AI`; } function updateGateLabel() { const label = document.getElementById('element2-copilot-gate'); if (!label) return; const status = gateStatus(); label.textContent = gateMessage(status); label.style.color = status.bothParties && status.audioReady && status.senderReady ? '#86efac' : '#fcd34d'; } function scoreAudioCandidate(item) { if (!item || !isLive(item.track) || item.track.muted || aiOutputTrackIds.has(item.track.id)) return -1; let score = 0; if (item.receiver) score += 300; if ((item.bytesReceived || 0) > 0) score += 100; if ((item.bytesDelta || 0) > 0) score += 4000 + Math.min(1500, item.bytesDelta); if ((item.packetsDelta || 0) > 0) score += 2000 + Math.min(800, item.packetsDelta * 8); score += Math.min(1200, (item.audioLevel || 0) * 12000); if (elementAudioReasons(item).length > 0) score += 100; item.audioScore = score; return score; } function rememberRemoteReceiver(receiver, reason) { const track = receiver?.track; if (!isLive(track) || isInternalPc(receiver?.transport)) return; if (track.kind === 'audio' && !aiOutputTrackIds.has(track.id)) { let item = remoteAudio.get(track.id); if (!item) { item = { track, receiver, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 }; remoteAudio.set(track.id, item); log(`audio candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.receiver = receiver || item.receiver; item.reasons.add(reason); selectRemoteAudio(reason); } if (track.kind === 'video') { let item = remoteVideo.get(track.id); if (!item) { item = { track, receiver, reasons: new Set() }; remoteVideo.set(track.id, item); log(`video candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.receiver = receiver || item.receiver; item.reasons.add(reason); selectRemoteVideo(reason); } } function rememberRemoteStream(stream, reason) { if (!stream || !(stream instanceof MediaStream)) return; for (const track of stream.getAudioTracks()) { if (aiOutputTrackIds.has(track.id)) continue; let item = remoteAudio.get(track.id); if (!item) { item = { track, receiver: null, reasons: new Set(), bytesDelta: 0, packetsDelta: 0, audioLevel: 0, audioScore: 0 }; remoteAudio.set(track.id, item); log(`audio stream candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.reasons.add(reason); } for (const track of stream.getVideoTracks()) { let item = remoteVideo.get(track.id); if (!item) { item = { track, receiver: null, reasons: new Set() }; remoteVideo.set(track.id, item); log(`video stream candidate ${trackLabel(track)} (${reason})`, true); } item.track = track; item.reasons.add(reason); } selectRemoteAudio(reason); selectRemoteVideo(reason); } function selectRemoteAudio(reason) { const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted && hasElementReason(current, STATIC_AUDIO_ELEMENT_REASON)) { updateGateLabel(); return selectedRemoteAudio; } const item = staticAudioItem(); if (!item) { updateSelectedAudioLabel(); updateGateLabel(); return null; } if (selectedRemoteAudio !== item.track) { selectedRemoteAudio = item.track; log(`selected input audio (${reason}) from ${STATIC_AUDIO_ELEMENT_REASON}: ${trackLabel(selectedRemoteAudio)}`, true); updateSelectedAudioLabel(); attachAudioToAi(); } updateGateLabel(); return selectedRemoteAudio; } function selectRemoteVideo(reason) { const current = selectedRemoteVideo && remoteVideo.get(selectedRemoteVideo.id); if (isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted && hasElementReason(current, STATIC_VIDEO_ELEMENT_REASON)) { return selectedRemoteVideo; } const item = Array.from(remoteVideo.values()).find((candidate) => ( isLive(candidate.track) && !candidate.track.muted && hasElementReason(candidate, STATIC_VIDEO_ELEMENT_REASON) )); if (!item) return null; if (selectedRemoteVideo !== item.track) { selectedRemoteVideo = item.track; log(`selected input video (${reason}) from ${STATIC_VIDEO_ELEMENT_REASON}: ${trackLabel(selectedRemoteVideo)}`, true); attachVideoToAi(); } return selectedRemoteVideo; } function rememberBusinessSender(sender, track, reason) { if (!sender) return; const senderTrack = track || sender.track; if (senderTrack && senderTrack.kind !== 'audio') return; const id = sender.__aiElement2GatedSenderId || `sender-${businessSenders.size + 1}`; sender.__aiElement2GatedSenderId = id; let item = businessSenders.get(id); if (!item) { item = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesDelta: 0, packetsDelta: 0 }; businessSenders.set(id, item); log(`business sender candidate ${id} (${reason}): ${trackLabel(senderTrack)}`, true); } item.sender = sender; item.track = sender.track || senderTrack || item.track; if (!isAiTakeover && senderTrack && senderTrack !== aiAudioTrack) item.originalTrack = senderTrack; item.sources.add(reason); wrapBusinessSender(sender); selectBusinessSender(reason); } function wrapBusinessSender(sender) { if (!sender || sender.__aiElement2GatedWrapped) return; const originalReplaceTrack = sender.replaceTrack?.bind(sender); if (!originalReplaceTrack) return; sender.replaceTrack = async function (track) { const result = await originalReplaceTrack(track); rememberBusinessSender(sender, track, 'replaceTrack'); return result; }; sender.__aiElement2GatedWrapped = true; } function isRemoteInputTrack(track) { return Boolean(track && remoteAudio.has(track.id) && !aiOutputTrackIds.has(track.id)); } function selectBusinessSender(reason) { if (selectedBusinessSender && !isRemoteInputTrack(selectedBusinessSender.track)) return selectedBusinessSender; const item = Array.from(businessSenders.values()).find((candidate) => { const track = candidate.sender.track || candidate.track || candidate.originalTrack; return candidate.sources.has('pc-addTrack') && !isRemoteInputTrack(track); }); if (!item) return null; selectedBusinessSender = item.sender; originalBusinessTrack = item.originalTrack || item.sender.track || originalBusinessTrack; log(`selected output sender (${reason}): ${item.id}, track=${trackLabel(item.sender.track)}`, true); updateGateLabel(); attachAiOutputToBusiness(); return selectedBusinessSender; } function setVu(id, level) { const el = document.getElementById(id); if (el) el.style.width = `${Math.min(100, level * 2500)}%`; } function renderVu() { inputVu *= 0.88; outputVu *= 0.88; setVu('element2-copilot-input-vu', inputVu); setVu('element2-copilot-output-vu', outputVu); requestAnimationFrame(renderVu); } async function refreshStats() { await refreshInputStats(); await refreshOutputStats(); } async function refreshInputStats() { for (const item of remoteAudio.values()) { if (!item.receiver?.getStats) continue; const report = await item.receiver.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; const prev = inboundMemory.get(stat.id); const bytes = stat.bytesReceived || 0; const packets = stat.packetsReceived || 0; let audioLevel = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0; if ( prev && typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number' && typeof prev.energy === 'number' && typeof prev.duration === 'number' ) { const energyDelta = stat.totalAudioEnergy - prev.energy; const durationDelta = stat.totalSamplesDuration - prev.duration; if (energyDelta > 0 && durationDelta > 0) { audioLevel = Math.max(audioLevel, Math.sqrt(energyDelta / durationDelta)); } } item.bytesReceived = bytes; item.packetsReceived = packets; item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; item.audioLevel = audioLevel; inboundMemory.set(stat.id, { bytes, packets, energy: stat.totalAudioEnergy, duration: stat.totalSamplesDuration, }); if (item.track === selectedRemoteAudio) { inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500), Math.min(0.18, audioLevel)); } }); } selectRemoteAudio('stats'); } async function refreshOutputStats() { for (const item of businessSenders.values()) { if (!item.sender?.getStats) continue; item.track = item.sender.track || item.track; const report = await item.sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; const key = `${item.id}:${stat.id}`; const prev = outboundMemory.get(key); const bytes = stat.bytesSent || 0; const packets = stat.packetsSent || 0; item.bytesSent = bytes; item.packetsSent = packets; item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; outboundMemory.set(key, { bytes, packets }); if (item.sender === selectedBusinessSender) { outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); } }); } } async function outboundBytes(sender) { if (!sender?.getStats) return null; let total = 0; const report = await sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type === 'outbound-rtp' && kind === 'audio') total += stat.bytesSent || 0; }); return total; } function aiState() { if (!aiPc) return 'pc=null'; return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; } function aiReady() { return Boolean(aiPc?.remoteDescription && ( aiPc.connectionState === 'connected' || aiPc.iceConnectionState === 'connected' || aiPc.iceConnectionState === 'completed' )); } function waitForAiReady(timeoutMs = 10000) { if (aiReady()) return Promise.resolve(true); return new Promise((resolve) => { const startedAt = Date.now(); const timer = setInterval(() => { if (aiReady()) { clearInterval(timer); resolve(true); } else if (Date.now() - startedAt >= timeoutMs) { clearInterval(timer); resolve(false); } }, 250); }); } function checkAiInput(track) { if (inputHealthTimer) clearTimeout(inputHealthTimer); inputHealthTimer = setTimeout(async () => { if (!aiAudioSender || aiAudioSender.track !== track) return; await waitForAiReady(); const before = await outboundBytes(aiAudioSender); await new Promise((resolve) => setTimeout(resolve, 2500)); const after = await outboundBytes(aiAudioSender); log(`AI input packet check: before=${before}, after=${after}, ${aiState()}`, true); }, 500); } function checkAiOutput(track) { if (outputHealthTimer) clearTimeout(outputHealthTimer); outputHealthTimer = setTimeout(async () => { if (!selectedBusinessSender || selectedBusinessSender.track !== track) return; const before = await outboundBytes(selectedBusinessSender); await new Promise((resolve) => setTimeout(resolve, 2500)); const after = await outboundBytes(selectedBusinessSender); log(`AI output packet check: before=${before}, after=${after}, sender=${trackLabel(selectedBusinessSender.track)}`, true); }, 800); } async function attachAudioToAi() { if (!shouldSendAudioToAi()) return false; if (!aiAudioSender || !isLive(selectedRemoteAudio)) return false; if (aiAudioSender.track === selectedRemoteAudio) return true; await aiAudioSender.replaceTrack(selectedRemoteAudio); log(`AI input uses ${trackLabel(selectedRemoteAudio)} from ${STATIC_AUDIO_ELEMENT_REASON}`, true); checkAiInput(selectedRemoteAudio); return true; } async function attachVideoToAi() { if (!shouldSendVideoToAi()) return false; if (!aiVideoSender || !isLive(selectedRemoteVideo)) return false; if (aiVideoSender.track === selectedRemoteVideo) return true; await aiVideoSender.replaceTrack(selectedRemoteVideo); log(`AI video uses ${trackLabel(selectedRemoteVideo)} from ${STATIC_VIDEO_ELEMENT_REASON}`, true); return true; } async function attachAiOutputToBusiness() { selectBusinessSender('before-output'); if (!isAiTakeover || !selectedBusinessSender || !isLive(aiAudioTrack)) { log(`AI output waiting: takeover=${isAiTakeover}, sender=${Boolean(selectedBusinessSender)}, aiTrack=${trackLabel(aiAudioTrack)}`); return false; } if (selectedBusinessSender.track === aiAudioTrack) return true; await selectedBusinessSender.replaceTrack(aiAudioTrack); aiAttached = true; log(`business sender uses AI audio ${trackLabel(aiAudioTrack)}`, true); checkAiOutput(aiAudioTrack); return true; } async function restoreBusinessAudio() { if (!selectedBusinessSender || !aiAttached) return; await selectedBusinessSender.replaceTrack(originalBusinessTrack || null); aiAttached = false; log(`business sender restored ${trackLabel(originalBusinessTrack)}`, true); } function wrapTrackListener(listener) { if (!listener || listener.__aiElement2GatedWrapped) return listener; const wrapped = function (event) { const ownerPc = this instanceof RTCPeerConnection ? this : event?.currentTarget; if (!isInternalPc(ownerPc) && event?.track) { rememberRemoteReceiver(event.receiver, 'pc-track'); for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); } if (typeof listener === 'function') return listener.apply(this, arguments); if (listener?.handleEvent) return listener.handleEvent(event); return undefined; }; wrapped.__aiElement2GatedWrapped = true; return wrapped; } RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { if (type === 'track') return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); return originalAddEventListener.call(this, type, listener, options); }; try { const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { configurable: true, enumerable: true, get() { return descriptor?.get ? descriptor.get.call(this) : this.__aiElement2GatedOnTrack || null; }, set(listener) { const wrapped = wrapTrackListener(listener); this.__aiElement2GatedOnTrack = wrapped; if (descriptor?.set) descriptor.set.call(this, wrapped); else this.addEventListener('track', wrapped); }, }); } catch (error) { console.warn('[AI Element2 Gated] failed to wrap ontrack', error); } RTCPeerConnection.prototype.addTrack = function (track, ...streams) { const sender = originalAddTrack.call(this, track, ...streams); if (!isInternalPc(this) && track?.kind === 'audio') rememberBusinessSender(sender, track, 'pc-addTrack'); return sender; }; if (originalAddTransceiver) { RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { const transceiver = originalAddTransceiver.call(this, trackOrKind, init); if (!isInternalPc(this)) { const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; if (kind === 'audio') { const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; rememberBusinessSender(transceiver.sender, track, 'pc-addTransceiver'); } } return transceiver; }; } RTCPeerConnection.prototype.setRemoteDescription = function () { const result = originalSetRemoteDescription.apply(this, arguments); if (!isInternalPc(this)) { Promise.resolve(result).then(() => { for (const receiver of this.getReceivers?.() || []) rememberRemoteReceiver(receiver, 'receiver-after-srd'); for (const sender of this.getSenders?.() || []) rememberBusinessSender(sender, sender.track, 'sender-after-srd'); }).catch(() => {}); } return result; }; if (mediaSrcObjectDescriptor?.set) { Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { configurable: true, enumerable: mediaSrcObjectDescriptor.enumerable, get() { return mediaSrcObjectDescriptor.get.call(this); }, set(stream) { if (!this.__aiElement2GatedCopilotElement) { const reason = elementReasonFor(this) || `${this.tagName.toLowerCase()}-srcObject`; rememberRemoteStream(stream, reason); } return mediaSrcObjectDescriptor.set.call(this, stream); }, }); } function scanMediaElements() { document.querySelectorAll('audio').forEach((element, index) => { if (element.__aiElement2GatedCopilotElement) return; rememberRemoteStream(element.srcObject, `element-${index}:audio`); }); document.querySelectorAll('video').forEach((element, index) => { if (element.__aiElement2GatedCopilotElement) return; rememberRemoteStream(element.srcObject, `element-${index}:video`); }); selectRemoteAudio('media-scan'); selectRemoteVideo('media-scan'); updateGateLabel(); } setInterval(scanMediaElements, 1000); async function connectAi(signalingUrl, assistantId) { scanMediaElements(); selectRemoteAudio('connect'); selectRemoteVideo('connect'); selectBusinessSender('connect'); const status = gateStatus(); updateGateLabel(); if (!status.bothParties || !status.audioReady || !status.senderReady) { appendChatLine('门槛', gateMessage(status), true); setStatus('disconnected'); return false; } disconnectAi(); selectedRemoteAudio = status.track; const id = pcId(); aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); markInternalPc(aiPc); aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); aiDc = aiPc.createDataChannel('chat'); aiDc.onopen = () => { setStatus('connected'); log('AI data channel open', true); aiDc.send(JSON.stringify({ type: 'client-ready' })); }; aiDc.onclose = () => setStatus('disconnected'); aiDc.onmessage = (event) => { try { const msg = JSON.parse(event.data); const content = msg.delta || msg.content; if (msg.type === 'transcript') { logUserTranscript(content); } else if (msg.type === 'assistant-text-start') { startAssistantLine(); } else if (msg.type === 'assistant-text-delta') { appendAssistantDelta(content); } else if (msg.type === 'assistant-text-end') { finishAssistantLine(); } else { log(`data channel message: ${msg.type}`); } } catch { appendChatLine('DATA', event.data); } }; const sendAudio = shouldSendAudioToAi(); const audioTransceiver = sendAudio && isLive(selectedRemoteAudio) ? aiPc.addTransceiver(selectedRemoteAudio, { direction: 'sendrecv' }) : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); aiAudioSender = audioTransceiver.sender; log(!sendAudio ? 'AI audio uplink disabled by panel setting' : isLive(selectedRemoteAudio) ? `AI input starts with ${trackLabel(selectedRemoteAudio)} (${STATIC_AUDIO_ELEMENT_REASON})` : `AI input waiting for ${STATIC_AUDIO_ELEMENT_REASON}`, true); const sendVideo = shouldSendVideoToAi(); const shouldSendVideo = sendVideo && isLive(selectedRemoteVideo) && !selectedRemoteVideo.muted; if (shouldSendVideo) { const videoTransceiver = aiPc.addTransceiver(selectedRemoteVideo, { direction: 'sendonly' }); aiVideoSender = videoTransceiver.sender; log(`AI video starts with ${trackLabel(selectedRemoteVideo)} (${STATIC_VIDEO_ELEMENT_REASON})`); } else { aiVideoSender = null; log(sendVideo ? `AI video disabled: waiting for live track at ${STATIC_VIDEO_ELEMENT_REASON}` : 'AI video uplink disabled by panel setting'); } aiPc.ontrack = (event) => { if (event.track.kind !== 'audio') return; aiOutputTrackIds.add(event.track.id); remoteAudio.delete(event.track.id); aiAudioTrack = event.track; log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); aiAudioTrack.onunmute = () => { log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); attachAiOutputToBusiness(); }; attachAiOutputToBusiness(); }; aiPc.onicecandidate = (event) => { if (aiWs?.readyState !== WebSocket.OPEN) return; aiWs.send(JSON.stringify({ type: 'ice-candidate', payload: { pc_id: id, candidate: event.candidate ? { candidate: event.candidate.candidate, sdpMid: event.candidate.sdpMid, sdpMLineIndex: event.candidate.sdpMLineIndex, } : null, }, })); }; setStatus('connecting'); aiWs = new WebSocket(signalingUrl); aiWs.onopen = async () => { const offer = await aiPc.createOffer(); await aiPc.setLocalDescription(offer); aiWs.send(JSON.stringify({ type: 'offer', payload: { pc_id: id, sdp: aiPc.localDescription.sdp, type: aiPc.localDescription.type, assistant_id: assistantId, vision_enabled: shouldSendVideo, }, })); log('sent AI WebRTC offer', true); }; aiWs.onmessage = async (event) => { try { const msg = JSON.parse(event.data); if (msg.type === 'answer') { await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); log(`AI WebRTC answer applied; ${aiState()}`, true); } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); } else if (msg.type === 'error') { throw new Error(msg.payload?.message || 'server error'); } } catch (error) { log(`signaling error: ${error.message || error}`); } }; aiWs.onerror = () => { setStatus('failed'); log('AI signaling websocket error'); }; aiWs.onclose = () => setStatus('disconnected'); attachAudioToAi(); attachVideoToAi(); return true; } function disconnectAi() { try { aiDc?.close(); } catch {} try { aiPc?.close(); } catch {} try { aiWs?.close(); } catch {} aiDc = null; aiPc = null; aiWs = null; aiAudioSender = null; aiVideoSender = null; aiAudioTrack = null; if (inputRecorder?.state === 'recording') stopInputRecording(); if (videoRecorder?.state === 'recording') stopVideoRecording(); setStatus('disconnected'); } function setStatus(status) { const el = document.getElementById('element2-copilot-status'); if (!el) return; el.textContent = { disconnected: '未连接', connecting: '连接中', connected: '已连接', failed: '连接失败', }[status] || status; } function updateModeButton() { const button = document.getElementById('element2-copilot-toggle'); if (!button) return; if (isAiTakeover) { button.textContent = '当前:AI 接管|点击切回人工'; button.style.background = '#8b5cf6'; } else { button.textContent = '当前:人工接管|点击切换 AI'; button.style.background = '#10b981'; } } window.__aiElement2GatedDump = function () { const status = gateStatus(); return { audioSelection: STATIC_AUDIO_ELEMENT_REASON, staticVideoElement: STATIC_VIDEO_ELEMENT_REASON, gate: { bothParties: status.bothParties, audioReady: status.audioReady, senderReady: status.senderReady, track: trackLabel(status.track), }, gateMessage: gateMessage(status), selectedInput: trackLabel(selectedRemoteAudio), selectedVideo: trackLabel(selectedRemoteVideo), aiVideoSenderTrack: trackLabel(aiVideoSender?.track), selectedOutput: trackLabel(selectedBusinessSender?.track), originalOutput: trackLabel(originalBusinessTrack), aiAudio: trackLabel(aiAudioTrack), aiAttached, aiState: aiState(), recording: inputRecorder?.state === 'recording' ? { track: trackLabel(inputRecordingTrack), seconds: ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1), chunks: inputRecordingChunks.length, } : null, videoRecording: videoRecorder?.state === 'recording' ? { track: trackLabel(videoRecordingTrack), seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1), chunks: videoRecordingChunks.length, } : null, inputs: Array.from(remoteAudio.values()).map((item) => ({ selected: item.track === selectedRemoteAudio, id: item.track.id, muted: item.track.muted, live: item.track.readyState, hasReceiver: Boolean(item.receiver), source: audioSourceLabel(item), bytesReceived: item.bytesReceived || 0, packetsReceived: item.packetsReceived || 0, bytesDelta: item.bytesDelta, packetsDelta: item.packetsDelta, audioLevel: item.audioLevel || 0, audioScore: Math.round(item.audioScore || scoreAudioCandidate(item) || 0), matchesStaticElement: hasElementReason(item, STATIC_AUDIO_ELEMENT_REASON), reasons: Array.from(item.reasons), })), videos: Array.from(remoteVideo.values()).map((item) => ({ selected: item.track === selectedRemoteVideo, id: item.track.id, muted: item.track.muted, live: item.track.readyState, hasReceiver: Boolean(item.receiver), matchesStaticElement: hasElementReason(item, STATIC_VIDEO_ELEMENT_REASON), reasons: Array.from(item.reasons), })), outputs: Array.from(businessSenders.values()).map((item) => ({ selected: item.sender === selectedBusinessSender, id: item.id, track: trackLabel(item.sender.track || item.track), originalTrack: trackLabel(item.originalTrack), pcAddTrack: item.sources.has('pc-addTrack'), isRemoteInput: isRemoteInputTrack(item.sender.track || item.track), bytesSent: item.bytesSent, packetsSent: item.packetsSent, bytesDelta: item.bytesDelta, packetsDelta: item.packetsDelta, sources: Array.from(item.sources), })), }; }; function renderPanel() { if (!document.body || document.getElementById('element2-copilot-panel')) return; const panel = document.createElement('div'); panel.id = 'element2-copilot-panel'; panel.style.cssText = [ 'position:fixed', 'top:20px', 'left:20px', 'width:360px', 'z-index:999999', 'box-sizing:border-box', 'padding:16px', 'border-radius:10px', 'background:rgba(17,24,39,.96)', 'color:white', 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', 'box-shadow:0 12px 32px rgba(0,0,0,.35)', ].join(';'); panel.innerHTML = `