// ==UserScript== // @name 12345 AI 协同中台 // @namespace http://tampermonkey.net/ // @version 7.5 // @description 引入 PCM ScriptProcessor 强制软解防丢包Bug;增加真实洗流耳返监听功能,彻底解决远端触发失败悬案 // @author You // @match *://yhy-yw.22sj.cn:*/* // @grant none // @run-at document-start // ==/UserScript== (function() { 'use strict'; console.log("🚀 [AI Copilot WebRTC] 注入脚本已加载,采用洗流桥接架构 v7.5 (强制 PCM 软解+耳返版)..."); // ========================================== // 1. 全局状态与 WebRTC 核心对象 // ========================================== let originalGetUserMedia = null; if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) { originalGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices); } const origAddEventListener = RTCPeerConnection.prototype.addEventListener; const origSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; const origAddTrack = RTCPeerConnection.prototype.addTrack; const origAddTransceiver = RTCPeerConnection.prototype.addTransceiver; const origAddStream = RTCPeerConnection.prototype.addStream; const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); let audioCtx = null; let meterAudioCtx = null; let meterUploadDestination = null; let micGainNode = null; let aiOutGainNode = null; let aiLocalGainNode = null; let washedUploadTrack = null; let keepAliveOscillator = null; let monitorGainNode = null; // 耳返监听控制器 let pc = null; let wsSignaling = null; let dc = null; let audioSender = null; let videoSender = null; let analyserNode = null; let businessAudioSender = null; let businessOriginalAudioTrack = null; let aiRemoteAudioTrack = null; let aiTrackAttachedToBusiness = false; let aiInputMode = 'washed'; let directInputHealthTimer = null; let directInputTrackId = null; let isAiTakeover = false; let isMonitoring = false; window.__allStolenStreams = new Set(); window.__ignoredTrackIds = new Set(); let currentStolenAudioTrack = null; let currentStolenVideoTrack = null; let currentMeterSourceNode = null; let currentVadBoostGainNode = null; let meterKeepAliveGainNode = null; let pendingBridgeAudioTrack = null; let statsVuLevel = 0; const remoteReceiversByTrackId = new Map(); const receiverStatsMemory = new Map(); const capturedVideoElementStreams = new WeakMap(); function createAudioContext() { const AudioContextClass = window.AudioContext || window.webkitAudioContext; try { return new AudioContextClass({ sampleRate: 48000 }); } catch (e) { console.warn("48k AudioContext 创建失败,回退浏览器默认采样率:", e); return new AudioContextClass(); } } function isInternalPeerConnection(target) { return !!(target && target.__aiCopilotInternal); } function generatePcId() { const bytes = new Uint8Array(16); crypto.getRandomValues(bytes); return "PC-" + Array.from(bytes).map(b => b.toString(16).padStart(2, "0")).join(""); } function isSafeToSteal(stream) { if (!stream || !(stream instanceof MediaStream)) return false; if (stream.__internal_bypass) return false; if (stream.getTracks().length === 0) return false; for (const track of stream.getTracks()) { if (window.__ignoredTrackIds.has(track.id)) return false; } return true; } function rememberRemoteStream(stream, reason = 'unknown') { if (!isSafeToSteal(stream)) return; window.__allStolenStreams.add(stream); console.log(`[AI Copilot WebRTC] 捕获候选远端流(${reason})`, stream.id, stream.getTracks().map(t => `${t.kind}:${t.readyState}:${t.id}`)); } function rememberRemoteTrack(track, stream, reason = 'track') { if (!track || window.__ignoredTrackIds.has(track.id)) return; const targetStream = isSafeToSteal(stream) ? stream : new MediaStream([track]); rememberRemoteStream(targetStream, reason); } function rememberRemoteReceiver(track, receiver, reason = 'receiver') { if (!track || !receiver || window.__ignoredTrackIds.has(track.id)) return; remoteReceiversByTrackId.set(track.id, receiver); console.log(`[AI Copilot WebRTC] 绑定远端 receiver(${reason})`, track.kind, track.id); } function rememberBusinessAudioSender(sender, track, reason = 'unknown') { if (!sender) return; const senderTrack = track || sender.track; if (senderTrack && senderTrack.kind !== 'audio') return; businessAudioSender = sender; if (senderTrack && senderTrack !== aiRemoteAudioTrack) { businessOriginalAudioTrack = senderTrack; } console.log(`[AI Copilot WebRTC] 绑定业务通话 audio sender(${reason})`, senderTrack?.id || 'no-track-yet'); if (isAiTakeover && aiRemoteAudioTrack) attachAiTrackToBusiness(); } function wrapBusinessSender(sender) { if (!sender || sender.__aiCopilotWrappedSender) return sender; const originalReplaceTrack = sender.replaceTrack?.bind(sender); if (!originalReplaceTrack) return sender; sender.replaceTrack = async function(track) { const ret = await originalReplaceTrack(track); if (track && track.kind === 'audio' && track !== aiRemoteAudioTrack && !isAiTakeover) { rememberBusinessAudioSender(sender, track, 'sender-replaceTrack'); } return ret; }; sender.__aiCopilotWrappedSender = true; return sender; } async function attachAiTrackToBusiness() { if (!isAiTakeover || !businessAudioSender || !aiRemoteAudioTrack || aiRemoteAudioTrack.readyState !== 'live') return false; try { await businessAudioSender.replaceTrack(aiRemoteAudioTrack); aiTrackAttachedToBusiness = true; appendLog("🔁 已将 AI 原生音轨直接替换到业务通话上行", true); return true; } catch (e) { aiTrackAttachedToBusiness = false; appendLog(`⚠️ AI 原生音轨替换业务 sender 失败: ${e.message || e}`); return false; } } async function restoreBusinessAudioTrack() { if (!businessAudioSender || !aiTrackAttachedToBusiness) return; try { await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null); appendLog("👨‍💼 已恢复业务通话原始音轨", true); } catch (e) { appendLog(`⚠️ 恢复业务音轨失败: ${e.message || e}`); } finally { aiTrackAttachedToBusiness = false; } } async function outboundAudioBytes(sender) { if (!sender?.getStats) return null; try { let total = 0; const report = await sender.getStats(); report.forEach(stat => { const statKind = stat.kind || stat.mediaType; if (stat.type === 'outbound-rtp' && statKind === 'audio' && typeof stat.bytesSent === 'number') { total += stat.bytesSent; } }); return total; } catch (e) { return null; } } function aiPeerReadyForMedia() { if (!pc || !pc.remoteDescription) return false; return ( pc.connectionState === 'connected' || pc.iceConnectionState === 'connected' || pc.iceConnectionState === 'completed' ); } function aiPeerStateLabel() { if (!pc) return 'pc=null'; return `conn=${pc.connectionState},ice=${pc.iceConnectionState},signaling=${pc.signalingState},remote=${pc.remoteDescription?.type || 'none'}`; } function waitForAiPeerMediaReady(timeoutMs = 10000) { if (aiPeerReadyForMedia()) return Promise.resolve(true); return new Promise((resolve) => { const startedAt = Date.now(); const timer = setInterval(() => { if (aiPeerReadyForMedia()) { clearInterval(timer); resolve(true); } else if (Date.now() - startedAt >= timeoutMs) { clearInterval(timer); resolve(false); } }, 250); }); } function disconnectWashedInputGraph() { if (currentMeterSourceNode) { try { currentMeterSourceNode.disconnect(); } catch(e) {} currentMeterSourceNode = null; } if (currentVadBoostGainNode) { try { currentVadBoostGainNode.disconnect(); } catch(e) {} currentVadBoostGainNode = null; } if (pcmForcerNode) { try { pcmForcerNode.disconnect(); } catch(e) {} } } async function fallbackToWashedAiInput(track, reason = 'fallback') { if (!track || track.readyState !== 'live') return; if (directInputHealthTimer) { clearTimeout(directInputHealthTimer); directInputHealthTimer = null; } directInputTrackId = null; if (audioSender && washedUploadTrack && audioSender.track !== washedUploadTrack) { try { await audioSender.replaceTrack(washedUploadTrack); } catch (e) { appendLog(`⚠️ AI 输入切回洗流轨失败: ${e.message || e}`); } } aiInputMode = 'washed'; appendLog(`🧯 AI 输入切回洗流链路(${reason})`, true); ensureVuMeterForTrack(track, reason); } async function tryDirectAiInput(track, reason = 'direct') { if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return false; if (!audioSender) { pendingBridgeAudioTrack = track; return false; } if (aiInputMode === 'direct' && directInputTrackId === track.id && audioSender.track === track) return true; try { if (directInputHealthTimer) clearTimeout(directInputHealthTimer); await audioSender.replaceTrack(track); aiInputMode = 'direct'; directInputTrackId = track.id; disconnectWashedInputGraph(); appendLog(`🚀 AI 输入直连远端原生音轨(${reason}); track=${track.id}, ${aiPeerStateLabel()}`, true); directInputHealthTimer = setTimeout(async () => { if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return; appendLog(`🩺 等待 AI WebRTC 媒体连接后检查直连输入: ${aiPeerStateLabel()}`); const ready = await waitForAiPeerMediaReady(10000); if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return; if (!ready) { appendLog(`⚠️ AI WebRTC 媒体连接超时,直连输入无法验证: ${aiPeerStateLabel()}`); await fallbackToWashedAiInput(track, 'direct-ai-pc-not-ready'); return; } const beforeHealthBytes = await outboundAudioBytes(audioSender); await new Promise(resolve => setTimeout(resolve, 2500)); if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return; const afterBytes = await outboundAudioBytes(audioSender); appendLog(`🩺 直连输入发包检查: before=${beforeHealthBytes}, after=${afterBytes}, ${aiPeerStateLabel()}`); if ( afterBytes !== null && ( (beforeHealthBytes !== null && afterBytes <= beforeHealthBytes) || (beforeHealthBytes === null && afterBytes === 0) ) ) { await fallbackToWashedAiInput(track, 'direct-no-outbound-audio'); } }, 500); return true; } catch (e) { appendLog(`⚠️ AI 输入直连失败,改用洗流: ${e.message || e}`); await fallbackToWashedAiInput(track, 'direct-replace-failed'); return false; } } function initMeterAudioCtx() { if (meterAudioCtx) return; try { meterAudioCtx = createAudioContext(); meterKeepAliveGainNode = meterAudioCtx.createGain(); meterKeepAliveGainNode.gain.value = 0; meterKeepAliveGainNode.connect(meterAudioCtx.destination); meterUploadDestination = meterAudioCtx.createMediaStreamDestination(); meterUploadDestination.stream.__internal_bypass = true; washedUploadTrack = meterUploadDestination.stream.getAudioTracks()[0]; window.__ignoredTrackIds.add(washedUploadTrack.id); analyserNode = meterAudioCtx.createAnalyser(); analyserNode.fftSize = 256; analyserNode.smoothingTimeConstant = 0.35; // 耳返监听节点,默认为静音 (0) monitorGainNode = meterAudioCtx.createGain(); monitorGainNode.gain.value = 0; monitorGainNode.connect(meterAudioCtx.destination); keepAliveOscillator = meterAudioCtx.createOscillator(); const oscGain = meterAudioCtx.createGain(); oscGain.gain.value = 0.0001; keepAliveOscillator.connect(oscGain); oscGain.connect(meterUploadDestination); keepAliveOscillator.start(); console.log("🛠️ 洗流音频总线与保活振荡器已创建"); } catch(e) { console.warn("VU 探测链路初始化失败:", e); } } async function bridgeRemoteAudioTrack(track, reason = 'scan') { if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return; if ( track === currentStolenAudioTrack && ( (aiInputMode === 'direct' && audioSender?.track === track) || (aiInputMode === 'washed' && currentMeterSourceNode) ) ) return; pendingBridgeAudioTrack = track; appendLog(`🎙️ 锁定远端发声轨(${reason}),尝试直连 AI 输入`, true); currentStolenAudioTrack = track; const directOk = await tryDirectAiInput(track, reason); if (!directOk && audioSender) { await fallbackToWashedAiInput(track, `${reason}-fallback`); } } function capturePlayableElementVideoStream(el) { if (!el || el.__aiCopilotInternalElement) return null; if (el.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return null; if (el.tagName !== 'VIDEO') return null; const existing = capturedVideoElementStreams.get(el); if (existing && existing.getVideoTracks().some(t => t.readyState === 'live')) return existing; const capture = el.captureStream || el.mozCaptureStream; if (typeof capture !== 'function') return null; try { const stream = capture.call(el); const liveVideoTracks = stream?.getVideoTracks().filter(t => t.readyState === 'live') || []; if (liveVideoTracks.length === 0) return null; stream.__internal_bypass = true; capturedVideoElementStreams.set(el, stream); appendLog("📹 已从页面播放元素 captureStream 捕获远端视频画面", true); return stream; } catch (e) { console.warn("captureStream 捕获播放元素视频失败:", e); return null; } } // ========================================== // 🛑 核心神技:PCM Passthrough 强制阻断 Chrome 硬件流 Bug // ========================================== let pcmForcerNode = null; function ensureVuMeterForTrack(track, reason = 'unknown') { if (!track || track.readyState !== 'live') return; initMeterAudioCtx(); try { if (currentMeterSourceNode) { try { currentMeterSourceNode.disconnect(); } catch(e) {} } if (currentVadBoostGainNode) { try { currentVadBoostGainNode.disconnect(); } catch(e) {} } if (pcmForcerNode) { try { pcmForcerNode.disconnect(); } catch(e) {} } const meterStream = new MediaStream([track]); meterStream.__internal_bypass = true; // 1. 回归 MediaStreamSource,但用 ScriptProcessor 打断它 currentMeterSourceNode = meterAudioCtx.createMediaStreamSource(meterStream); // 2. AGC 放大增益 (击穿 VAD 阈值) const vadBoostGain = meterAudioCtx.createGain(); vadBoostGain.gain.value = 2.0; // fallback 洗流仅轻度放大,避免底噪/回声被过度抬高 currentVadBoostGainNode = vadBoostGain; currentMeterSourceNode.connect(vadBoostGain); // 3. 终极杀招:PCM 强刷节点。迫使浏览器离开底层 WebRTC 硬件直通层,进入 JS 内存读取浮点数! // 缓存大小 2048/4096 皆可。1进1出。 pcmForcerNode = meterAudioCtx.createScriptProcessor(4096, 1, 1); pcmForcerNode.onaudioprocess = function(audioProcessingEvent) { const inputBuffer = audioProcessingEvent.inputBuffer; const outputBuffer = audioProcessingEvent.outputBuffer; // 将输入通道的 float32 数据原封不动写到输出通道 const inputData = inputBuffer.getChannelData(0); const outputData = outputBuffer.getChannelData(0); for (let i = 0; i < inputBuffer.length; i++) { outputData[i] = inputData[i]; } }; // 路由:源 -> 放大器 -> PCM强刷节点 -> (VU表 / WebRTC发送端 / 耳返节点 / 垃圾桶) vadBoostGain.connect(pcmForcerNode); pcmForcerNode.connect(analyserNode); pcmForcerNode.connect(meterUploadDestination); pcmForcerNode.connect(monitorGainNode); // 必须连接到音频输出才能触发 onaudioprocess,用 KeepAliveGainNode(增益为0) 当作垃圾桶 pcmForcerNode.connect(meterKeepAliveGainNode); window.__aiCopilotMeterSource = currentMeterSourceNode; if (meterAudioCtx.state === 'suspended') { meterAudioCtx.resume().catch(e => console.warn("VU 音频上下文唤醒失败:", e)); } appendLog(`📈 洗流管道已接管,注入 PCM 强刷节点(${reason})`, true); } catch(e) { console.warn("接入混流总线失败:", e); } } function armAudioContextResume() { const resume = () => { [meterAudioCtx, audioCtx].forEach(ctx => { if (ctx && ctx.state === 'suspended') ctx.resume().catch(() => {}); }); }; document.addEventListener('pointerdown', resume, true); document.addEventListener('keydown', resume, true); document.addEventListener('touchstart', resume, true); } armAudioContextResume(); function bridgeRemoteVideoTrack(track) { if (!track || track.readyState !== 'live' || track === currentStolenVideoTrack || window.__ignoredTrackIds.has(track.id)) return; currentStolenVideoTrack = track; appendLog(`📸 锁定远端视频,开启视觉推流`, true); const indicator = document.getElementById('video-status-indicator'); if (indicator) indicator.style.display = 'block'; if (videoSender) videoSender.replaceTrack(currentStolenVideoTrack).catch(e => console.warn(e)); } function appendLog(message, isImportant = false) { const logContainer = document.getElementById('ws-log-container'); if (!logContainer) return; if (logContainer.children.length === 1 && logContainer.children[0].innerText.includes('等待 DataChannel')) { logContainer.innerHTML = ''; } const logEntry = document.createElement('div'); logEntry.style.marginBottom = "4px"; if (isImportant) logEntry.style.color = "#fcd34d"; const time = new Date().toLocaleTimeString([], {hour12: false}); logEntry.textContent = `[${time}] ${message}`; logContainer.appendChild(logEntry); if (logContainer.children.length > 100) { logContainer.removeChild(logContainer.firstChild); } logContainer.scrollTop = logContainer.scrollHeight; } // VU 表渲染 let vuDataArray = null; function renderVU() { let rms = 0; if (analyserNode) { if (!vuDataArray || vuDataArray.length !== analyserNode.frequencyBinCount) { vuDataArray = new Uint8Array(analyserNode.frequencyBinCount); } analyserNode.getByteTimeDomainData(vuDataArray); let sumSquare = 0; for(let i = 0; i < vuDataArray.length; i++) { const val = (vuDataArray[i] - 128) / 128; sumSquare += val * val; } rms = Math.sqrt(sumSquare / vuDataArray.length); } rms = Math.max(rms, statsVuLevel); statsVuLevel *= 0.88; const vuBar = document.getElementById('upload-vu-bar'); if (vuBar) { const width = Math.min(100, rms * 2500); vuBar.style.width = width + '%'; } requestAnimationFrame(renderVU); } requestAnimationFrame(renderVU); setInterval(async () => { const receivers = Array.from(remoteReceiversByTrackId.values()); let bestLevel = 0; for (const receiver of receivers) { try { if (!receiver.track || receiver.track.kind !== 'audio' || receiver.track.readyState !== 'live') continue; const report = await receiver.getStats(); report.forEach(stat => { const statKind = stat.kind || stat.mediaType; if (stat.type !== 'inbound-rtp' || statKind !== 'audio') return; if (typeof stat.audioLevel === 'number') { bestLevel = Math.max(bestLevel, stat.audioLevel); } if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') { const prev = receiverStatsMemory.get(stat.id); receiverStatsMemory.set(stat.id, { energy: stat.totalAudioEnergy, duration: stat.totalSamplesDuration }); if (prev) { const energyDelta = stat.totalAudioEnergy - prev.energy; const durationDelta = stat.totalSamplesDuration - prev.duration; if (energyDelta > 0 && durationDelta > 0) { bestLevel = Math.max(bestLevel, Math.sqrt(energyDelta / durationDelta)); } } } }); } catch(e) {} } if (bestLevel > 0) statsVuLevel = Math.max(statsVuLevel, bestLevel); }, 250); // ========================================== // 2. 精准声源窃取拦截器 // ========================================== function wrapTrackListener(target, listener) { if (!listener || listener.__aiCopilotWrappedTrackListener) return listener; const wrapped = function(event) { if (!isInternalPeerConnection(this) && event && event.track) { rememberRemoteTrack(event.track, event.streams && event.streams[0], 'pc-track'); rememberRemoteReceiver(event.track, event.receiver, 'pc-track'); } if (typeof listener === 'function') return listener.apply(this, arguments); if (typeof listener === 'object' && listener.handleEvent) return listener.handleEvent(event); }; wrapped.__aiCopilotWrappedTrackListener = true; wrapped.__aiCopilotOriginalTrackListener = listener; return wrapped; } function wrapAddStreamListener(listener) { if (!listener || listener.__aiCopilotWrappedAddStreamListener) return listener; const wrapped = function(event) { if (!isInternalPeerConnection(this) && event && event.stream) { rememberRemoteStream(event.stream, 'pc-addstream'); } if (typeof listener === 'function') return listener.apply(this, arguments); if (typeof listener === 'object' && listener.handleEvent) return listener.handleEvent(event); }; wrapped.__aiCopilotWrappedAddStreamListener = true; return wrapped; } RTCPeerConnection.prototype.addEventListener = function(type, listener, options) { if (type === 'track') { return origAddEventListener.call(this, type, wrapTrackListener(this, listener), options); } if (type === 'addstream') { return origAddEventListener.call(this, type, wrapAddStreamListener(listener), options); } return origAddEventListener.call(this, type, listener, options); }; RTCPeerConnection.prototype.addTrack = function(track, ...streams) { const sender = origAddTrack.call(this, track, ...streams); if (!isInternalPeerConnection(this) && track?.kind === 'audio') { rememberBusinessAudioSender(wrapBusinessSender(sender), track, 'pc-addTrack'); } return sender; }; if (origAddTransceiver) { RTCPeerConnection.prototype.addTransceiver = function(trackOrKind, init) { const transceiver = origAddTransceiver.call(this, trackOrKind, init); if (!isInternalPeerConnection(this)) { const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : init?.sendEncodings ? transceiver.sender?.track : null; const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; if (kind === 'audio') { rememberBusinessAudioSender(wrapBusinessSender(transceiver.sender), track || transceiver.sender?.track, 'pc-addTransceiver'); } } return transceiver; }; } try { const onTrackDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { configurable: true, enumerable: true, get: function() { return onTrackDescriptor && onTrackDescriptor.get ? onTrackDescriptor.get.call(this) : this.__aiCopilotOnTrack || null; }, set: function(listener) { const wrapped = wrapTrackListener(this, listener); this.__aiCopilotOnTrack = wrapped; if (onTrackDescriptor && onTrackDescriptor.set) { onTrackDescriptor.set.call(this, wrapped); } else { this.addEventListener('track', wrapped); } } }); } catch (e) { console.warn("ontrack 拦截安装失败:", e); } try { const onAddStreamDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'onaddstream'); Object.defineProperty(RTCPeerConnection.prototype, 'onaddstream', { configurable: true, enumerable: true, get: function() { return onAddStreamDescriptor && onAddStreamDescriptor.get ? onAddStreamDescriptor.get.call(this) : this.__aiCopilotOnAddStream || null; }, set: function(listener) { const wrapped = wrapAddStreamListener(listener); this.__aiCopilotOnAddStream = wrapped; if (onAddStreamDescriptor && onAddStreamDescriptor.set) { onAddStreamDescriptor.set.call(this, wrapped); } else { this.addEventListener('addstream', wrapped); } } }); } catch (e) { console.warn("onaddstream 拦截安装失败:", e); } RTCPeerConnection.prototype.setRemoteDescription = function() { const ret = origSetRemoteDescription.apply(this, arguments); if (!isInternalPeerConnection(this)) { Promise.resolve(ret).then(() => { try { this.getReceivers?.().forEach(receiver => { if (receiver.track) { rememberRemoteTrack(receiver.track, new MediaStream([receiver.track]), 'receiver-after-srd'); rememberRemoteReceiver(receiver.track, receiver, 'receiver-after-srd'); } }); this.getRemoteStreams?.().forEach(stream => rememberRemoteStream(stream, 'remote-stream-after-srd')); } catch (e) {} }); } return ret; }; if (origAddStream) { RTCPeerConnection.prototype.addStream = function(stream) { if (!isInternalPeerConnection(this)) rememberRemoteStream(stream, 'pc-addStream-call'); return origAddStream.apply(this, arguments); }; } if (mediaSrcObjectDescriptor && mediaSrcObjectDescriptor.set) { Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { configurable: true, enumerable: mediaSrcObjectDescriptor.enumerable, get: function() { return mediaSrcObjectDescriptor.get.call(this); }, set: function(stream) { if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiCopilotInternalElement) { rememberRemoteStream(stream, 'media-srcObject'); } return mediaSrcObjectDescriptor.set.call(this, stream); } }); } const origPlay = HTMLMediaElement.prototype.play; HTMLMediaElement.prototype.play = function() { if (!this.muted && this.volume > 0 && isSafeToSteal(this.srcObject)) { rememberRemoteStream(this.srcObject, 'media-play'); } return origPlay.apply(this, arguments); }; const AudioContextCtor = window.AudioContext || window.webkitAudioContext; const origCreateMediaStreamSource = AudioContextCtor?.prototype?.createMediaStreamSource; if (AudioContextCtor && origCreateMediaStreamSource) { AudioContextCtor.prototype.createMediaStreamSource = function(stream) { if (isSafeToSteal(stream)) { rememberRemoteStream(stream, 'audio-context-source'); } return origCreateMediaStreamSource.call(this, stream); }; } setInterval(() => { document.querySelectorAll('audio, video').forEach(el => { if (!el.muted && el.volume > 0 && isSafeToSteal(el.srcObject)) { rememberRemoteStream(el.srcObject, 'media-scan'); } }); let foundVideoTrack = null; let foundAudioTrack = null; const mediaEls = document.querySelectorAll('audio, video'); for (let el of mediaEls) { if (el.__aiCopilotInternalElement) continue; if (!foundVideoTrack) { const capturedVideoStream = capturePlayableElementVideoStream(el); const capturedVideoTracks = capturedVideoStream?.getVideoTracks().filter(t => t.readyState === 'live') || []; if (capturedVideoTracks.length > 0) foundVideoTrack = capturedVideoTracks[0]; } if (!el.muted && el.volume > 0 && isSafeToSteal(el.srcObject)) { if (!foundVideoTrack) { const vTracks = el.srcObject.getVideoTracks().filter(t => t.readyState === 'live'); if (vTracks.length > 0) foundVideoTrack = vTracks[0]; } if (!foundAudioTrack) { const aTracks = el.srcObject.getAudioTracks().filter(t => t.readyState === 'live'); if (aTracks.length > 0) foundAudioTrack = aTracks[0]; } } } if (!foundAudioTrack || !foundVideoTrack) { for (const stream of Array.from(window.__allStolenStreams)) { if (!foundVideoTrack) { const videoTracks = stream.getVideoTracks().filter(t => t.readyState === 'live'); if (videoTracks.length > 0) foundVideoTrack = videoTracks[0]; } if (!foundAudioTrack) { const audioTracks = stream.getAudioTracks().filter(t => t.readyState === 'live'); if (audioTracks.length > 0) foundAudioTrack = audioTracks[0]; } if (foundVideoTrack && foundAudioTrack) break; } } bridgeRemoteVideoTrack(foundVideoTrack); bridgeRemoteAudioTrack(foundAudioTrack, 'scan').catch(e => console.warn(e)); }, 1000); // ========================================== // 3. 覆写 getUserMedia 构建本地混音路由 // ========================================== if (originalGetUserMedia) { navigator.mediaDevices.getUserMedia = async function (constraints) { if (constraints.video && !constraints.audio) { renderControlPanelSoon(); return originalGetUserMedia(constraints); } try { const realStream = await originalGetUserMedia(constraints); realStream.__internal_bypass = true; realStream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id)); audioCtx = createAudioContext(); micGainNode = null; aiOutGainNode = null; aiLocalGainNode = audioCtx.createGain(); aiLocalGainNode.gain.value = isAiTakeover ? 1.0 : 0; aiLocalGainNode.connect(audioCtx.destination); if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') { bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending').catch(e => console.warn(e)); } renderControlPanelSoon(); return realStream; } catch (err) { console.error("🚨 拦截失败:", err); return originalGetUserMedia(constraints); } }; } // ========================================== // 4. WebRTC 核心信令 // ========================================== async function connectWebRTC(wsUrl, assistantId) { disconnectWebRTC(); try { const pcId = generatePcId(); pc = new RTCPeerConnection({ iceServers: [{ urls: "stun:stun.l.google.com:19302" }] }); pc.__aiCopilotInternal = true; const statusEl = document.getElementById('ws-status'); if(statusEl) statusEl.textContent = "🟡 协商中..."; appendLog("🔄 正在建立 WebRTC 连接..."); pc.onconnectionstatechange = () => appendLog(`🔌 AI PC connectionState=${pc.connectionState}`); pc.oniceconnectionstatechange = () => appendLog(`🧊 AI PC iceConnectionState=${pc.iceConnectionState}`); initMeterAudioCtx(); const activeStream = new MediaStream([washedUploadTrack]); const audioTransceiver = pc.addTransceiver(washedUploadTrack, { direction: 'sendrecv', streams: [activeStream] }); audioSender = audioTransceiver.sender; appendLog("🎤 已挂载 AI 输入音频通道 (优先直连,失败回退洗流)", true); if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') { bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'sender-ready').catch(e => console.warn(e)); } const videoTransceiver = pc.addTransceiver('video', { direction: 'sendonly' }); videoSender = videoTransceiver.sender; if (currentStolenVideoTrack) { videoSender.replaceTrack(currentStolenVideoTrack).catch(e=>console.warn(e)); } pc.ontrack = (e) => { if (e.track.kind === 'audio' && audioCtx) { appendLog("🎧 接收到原生 AI 语音流,准备直连业务通话", true); window.__ignoredTrackIds.add(e.track.id); aiRemoteAudioTrack = e.track; const aiStream = new MediaStream([e.track]); aiStream.__internal_bypass = true; const aiStreamSource = audioCtx.createMediaStreamSource(aiStream); if (aiLocalGainNode) aiStreamSource.connect(aiLocalGainNode); window.__aiStreamSource = aiStreamSource; attachAiTrackToBusiness(); } }; dc = pc.createDataChannel("chat"); dc.onopen = () => { const statusEl = document.getElementById('ws-status'); if(statusEl) statusEl.textContent = "🟢 已连接"; appendLog("✅ WebRTC 数据通道建立成功"); dc.send(JSON.stringify({ type: "client-ready" })); }; dc.onclose = () => { const statusEl = document.getElementById('ws-status'); if(statusEl) statusEl.textContent = "🔴 未连接"; appendLog("❌ WebRTC 连接断开"); }; dc.onmessage = (event) => { try { const msg = JSON.parse(event.data); if (msg.type === 'assistant-text-delta' || msg.type === 'transcript') { const content = msg.delta || msg.content; if (content && content.trim()) appendLog(`💬 AI: ${content}`); } else if (msg.type === 'node-active') { appendLog(`🔄 工作流流转至节点: ${msg.nodeId}`, true); } else if (msg.type !== 'assistant-text-start' && msg.type !== 'assistant-text-end') { appendLog(`📥 收到信令: ${msg.type}`); } } catch (e) { appendLog(`📥 收到文本: ${event.data}`); } }; wsSignaling = new WebSocket(wsUrl); wsSignaling.onmessage = async (event) => { try { const msg = JSON.parse(event.data); if (msg.type === "answer") { await pc.setRemoteDescription({ type: "answer", sdp: msg.payload.sdp }); appendLog("📤 SDP 握手成功,底层引擎接管音视频传输", true); } else if (msg.type === "ice-candidate" && msg.payload?.candidate) { try { await pc.addIceCandidate(msg.payload.candidate); } catch(e){} } else if (msg.type === "error") { throw new Error(msg.payload?.message || "后端报错"); } } catch (e) { appendLog("⚠️ 信令处理错误: " + e.message); } }; wsSignaling.onopen = async () => { pc.onicecandidate = (e) => { if (wsSignaling.readyState !== WebSocket.OPEN) return; wsSignaling.send(JSON.stringify({ type: "ice-candidate", payload: { pc_id: pcId, candidate: e.candidate ? { candidate: e.candidate.candidate, sdpMid: e.candidate.sdpMid, sdpMLineIndex: e.candidate.sdpMLineIndex } : null } })); }; const offer = await pc.createOffer(); await pc.setLocalDescription(offer); appendLog("📹 已启用视频上行通道,检测到画面后会自动推送给后端", true); wsSignaling.send(JSON.stringify({ type: "offer", payload: { pc_id: pcId, sdp: pc.localDescription.sdp, type: pc.localDescription.type, assistant_id: assistantId, vision_enabled: true } })); }; wsSignaling.onerror = () => appendLog("⚠️ WebSocket 信令连接失败"); } catch (e) { console.error(e); appendLog("⚠️ 连接失败: " + e.message); const statusEl = document.getElementById('ws-status'); if (statusEl) statusEl.textContent = "🔴 连接失败"; } } function disconnectWebRTC() { if (directInputHealthTimer) { clearTimeout(directInputHealthTimer); directInputHealthTimer = null; } aiInputMode = 'washed'; directInputTrackId = null; if (dc) { try { dc.close(); } catch(e){} dc = null; } if (pc) { pc.close(); pc = null; } if (wsSignaling) { try { wsSignaling.close(); } catch(e){} wsSignaling = null; } const statusEl = document.getElementById('ws-status'); if (statusEl) statusEl.textContent = "🔴 未连接"; appendLog("⏹️ 已断开 WebRTC 引擎连接"); } // ========================================== // 5. 渲染控制面板 // ========================================== function renderControlPanel() { if (document.getElementById('ai-copilot-panel')) return; if (!document.body) return; const panel = document.createElement('div'); panel.id = 'ai-copilot-panel'; panel.style.cssText = ` position: fixed; top: 20px; left: 20px; background: rgba(17, 24, 39, 0.95); color: white; padding: 20px; border-radius: 12px; z-index: 999999; font-family: sans-serif; box-shadow: 0 10px 25px rgba(0,0,0,0.5); width: 340px; backdrop-filter: blur(10px); max-height: 90vh; overflow-y: auto; `; panel.innerHTML = `

🤖 AI WebRTC 引擎枢纽

🔴 未连接
当事人声音 (AI入站音量)

DataChannel 信令日志

等待 DataChannel 消息...
`; document.body.appendChild(panel); const btnToggleTakeover = document.getElementById('btn-toggle-takeover'); const btnTestText = document.getElementById('btn-test-text'); const btnTestVoice = document.getElementById('btn-test-voice'); const btnToggleMonitor = document.getElementById('btn-toggle-monitor'); const urlInput = document.getElementById('ws-url-input'); const assistantIdInput = document.getElementById('assistant-id-input'); // 耳返功能(直接排查法) btnToggleMonitor.addEventListener('click', () => { if (!monitorGainNode) return alert("总线尚未初始化!"); isMonitoring = !isMonitoring; monitorGainNode.gain.value = isMonitoring ? 1.0 : 0; if (isMonitoring) { btnToggleMonitor.textContent = "🎧 返听已开"; btnToggleMonitor.style.background = "#10b981"; appendLog("🎧 已开启硬件级返听,耳机听到的即是发给 AI 的声音", true); } else { btnToggleMonitor.textContent = "🎧 返听已关"; btnToggleMonitor.style.background = "#ef4444"; appendLog("🔇 返听已关闭"); } }); btnTestText.addEventListener('click', () => { if (!dc || dc.readyState !== 'open') { alert("⚠️ DataChannel 尚未连接,请先点击【AI 托管】。"); return; } const testText = "你好,你是谁"; dc.send(JSON.stringify({ type: "user-text", text: testText })); appendLog(`📝 发送强制测音文本: ${testText}`, true); }); btnTestVoice.addEventListener('click', () => { if (!meterAudioCtx || !meterUploadDestination) { alert("⚠️ 洗流总线未初始化,请先进行【AI 托管】连接。"); return; } if (meterAudioCtx.state === 'suspended') { meterAudioCtx.resume(); } const fileInput = document.createElement('input'); fileInput.type = 'file'; fileInput.accept = 'audio/*'; fileInput.style.display = 'none'; fileInput.onchange = (e) => { const file = e.target.files[0]; if (!file) { fileInput.remove(); return; } appendLog(`🔄 正在加载本地音频: ${file.name}...`); const blobUrl = URL.createObjectURL(file); const audioEl = new Audio(); audioEl.src = blobUrl; const fileSource = meterAudioCtx.createMediaElementSource(audioEl); fileSource.connect(meterUploadDestination); if (analyserNode) fileSource.connect(analyserNode); fileSource.connect(meterAudioCtx.destination); audioEl.play().then(() => { appendLog(`🗣️ 已成功将本地音频注入 WebRTC 音轨: ${file.name}`, true); }).catch(err => { appendLog(`❌ 注入音频播放失败: ${err.message}`); console.error("本地音频播放错误:", err); }); audioEl.onended = () => { URL.revokeObjectURL(blobUrl); appendLog(`⏹️ 本地音频播报完毕: ${file.name}`); }; fileInput.remove(); }; document.body.appendChild(fileInput); fileInput.click(); }); btnToggleTakeover.addEventListener('click', async () => { if (!audioCtx) { alert("音频路由未初始化!请先在网页内加入会议。"); return; } const asstId = assistantIdInput.value.trim(); if (!isAiTakeover && !asstId) { alert("请先填写 Assistant ID 再发起连接!"); return; } isAiTakeover = !isAiTakeover; const now = audioCtx.currentTime; if (isAiTakeover) { if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(1.0, now); btnToggleTakeover.textContent = "🤖 AI 托管中 (点击切回人工)"; btnToggleTakeover.style.background = "#8b5cf6"; if (!pc || pc.connectionState !== 'connected') { if (audioCtx && audioCtx.state === 'suspended') audioCtx.resume(); if (meterAudioCtx && meterAudioCtx.state === 'suspended') { meterAudioCtx.resume().catch(e => console.warn("meterAudioCtx 唤醒失败", e)); } connectWebRTC(urlInput.value, asstId); } if (!businessAudioSender) { appendLog("⚠️ 尚未捕获业务通话 audio sender,等待网页建立上行轨道"); } await attachAiTrackToBusiness(); } else { if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(0, now); await restoreBusinessAudioTrack(); btnToggleTakeover.textContent = "👨‍💼 人工通话中 (点击 AI 托管)"; btnToggleTakeover.style.background = "#10b981"; disconnectWebRTC(); } }); } function renderControlPanelSoon() { try { renderControlPanel(); } catch(e) {} } if (document.readyState === 'loading') { document.addEventListener('DOMContentLoaded', renderControlPanelSoon, { once: true }); } else { renderControlPanelSoon(); } setTimeout(renderControlPanelSoon, 1500); setTimeout(renderControlPanelSoon, 3000); // ========================================== // 🛑 UI 守护进程 // ========================================== setInterval(() => { if (document.body && !document.getElementById('ai-copilot-panel')) { console.log("⚠️ [AI Copilot] 检测到面板被网页框架覆盖,正在重新挂载..."); renderControlPanelSoon(); if (pc && pc.connectionState === 'connected') { const statusEl = document.getElementById('ws-status'); if (statusEl) statusEl.textContent = "🟢 已连接"; const btn = document.getElementById('btn-toggle-takeover'); if (btn && isAiTakeover) { btn.textContent = "🤖 AI 托管中 (点击切回人工)"; btn.style.background = "#8b5cf6"; } } } }, 2000); })();