diff --git a/scripts/copilot.js b/scripts/copilot.js index 8e98512..c5d53f6 100644 --- a/scripts/copilot.js +++ b/scripts/copilot.js @@ -52,11 +52,13 @@ let currentStolenAudioTrack = null; let currentStolenVideoTrack = null; let currentMeterSourceNode = null; + let currentVadBoostGainNode = null; let meterKeepAliveGainNode = null; let pendingBridgeAudioTrack = null; let statsVuLevel = 0; const remoteReceiversByTrackId = new Map(); const receiverStatsMemory = new Map(); + const capturedVideoElementStreams = new WeakMap(); function isInternalPeerConnection(target) { return !!(target && target.__aiCopilotInternal); @@ -142,6 +144,31 @@ ensureVuMeterForTrack(track, reason); } + function capturePlayableElementVideoStream(el) { + if (!el || el.__aiCopilotInternalElement) return null; + if (el.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return null; + if (el.tagName !== 'VIDEO') return null; + + const existing = capturedVideoElementStreams.get(el); + if (existing && existing.getVideoTracks().some(t => t.readyState === 'live')) return existing; + + const capture = el.captureStream || el.mozCaptureStream; + if (typeof capture !== 'function') return null; + + try { + const stream = capture.call(el); + const liveVideoTracks = stream?.getVideoTracks().filter(t => t.readyState === 'live') || []; + if (liveVideoTracks.length === 0) return null; + stream.__internal_bypass = true; + capturedVideoElementStreams.set(el, stream); + appendLog("📹 已从页面播放元素 captureStream 捕获远端视频画面", true); + return stream; + } catch (e) { + console.warn("captureStream 捕获播放元素视频失败:", e); + return null; + } + } + // ========================================== // 🛑 核心神技:PCM Passthrough 强制阻断 Chrome 硬件流 Bug // ========================================== @@ -154,6 +181,9 @@ if (currentMeterSourceNode) { try { currentMeterSourceNode.disconnect(); } catch(e) {} } + if (currentVadBoostGainNode) { + try { currentVadBoostGainNode.disconnect(); } catch(e) {} + } if (pcmForcerNode) { try { pcmForcerNode.disconnect(); } catch(e) {} } @@ -167,6 +197,7 @@ // 2. AGC 放大增益 (击穿 VAD 阈值) const vadBoostGain = meterAudioCtx.createGain(); vadBoostGain.gain.value = 8.0; // 放大 8 倍 + currentVadBoostGainNode = vadBoostGain; currentMeterSourceNode.connect(vadBoostGain); // 3. 终极杀招:PCM 强刷节点。迫使浏览器离开底层 WebRTC 硬件直通层,进入 JS 内存读取浮点数! @@ -463,6 +494,11 @@ const mediaEls = document.querySelectorAll('audio, video'); for (let el of mediaEls) { if (el.__aiCopilotInternalElement) continue; + if (!foundVideoTrack) { + const capturedVideoStream = capturePlayableElementVideoStream(el); + const capturedVideoTracks = capturedVideoStream?.getVideoTracks().filter(t => t.readyState === 'live') || []; + if (capturedVideoTracks.length > 0) foundVideoTrack = capturedVideoTracks[0]; + } if (!el.muted && el.volume > 0 && isSafeToSteal(el.srcObject)) { if (!foundVideoTrack) { const vTracks = el.srcObject.getVideoTracks().filter(t => t.readyState === 'live'); @@ -655,6 +691,7 @@ const offer = await pc.createOffer(); await pc.setLocalDescription(offer); + appendLog("📹 已启用视频上行通道,检测到画面后会自动推送给后端", true); wsSignaling.send(JSON.stringify({ type: "offer", payload: { @@ -662,7 +699,7 @@ sdp: pc.localDescription.sdp, type: pc.localDescription.type, assistant_id: assistantId, - vision_enabled: true + vision_enabled: true } })); }; @@ -930,4 +967,4 @@ } }, 2000); -})(); \ No newline at end of file +})();