Add video stream capture functionality in copilot.js

- Introduce a new function to capture video streams from playable HTML elements, enhancing the ability to process remote video feeds.
- Implement logic to manage and store captured video streams, ensuring only live tracks are utilized.
- Update existing audio processing functions to integrate video handling, improving overall media management in the WebRTC environment.
- Enhance logging for better debugging and user feedback regarding video stream capture status.
This commit is contained in:
Xin Wang
2026-07-08 00:40:57 +08:00
parent 2fb7399953
commit 97ac7ef79e

View File

@@ -52,11 +52,13 @@
let currentStolenAudioTrack = null; let currentStolenAudioTrack = null;
let currentStolenVideoTrack = null; let currentStolenVideoTrack = null;
let currentMeterSourceNode = null; let currentMeterSourceNode = null;
let currentVadBoostGainNode = null;
let meterKeepAliveGainNode = null; let meterKeepAliveGainNode = null;
let pendingBridgeAudioTrack = null; let pendingBridgeAudioTrack = null;
let statsVuLevel = 0; let statsVuLevel = 0;
const remoteReceiversByTrackId = new Map(); const remoteReceiversByTrackId = new Map();
const receiverStatsMemory = new Map(); const receiverStatsMemory = new Map();
const capturedVideoElementStreams = new WeakMap();
function isInternalPeerConnection(target) { function isInternalPeerConnection(target) {
return !!(target && target.__aiCopilotInternal); return !!(target && target.__aiCopilotInternal);
@@ -142,6 +144,31 @@
ensureVuMeterForTrack(track, reason); ensureVuMeterForTrack(track, reason);
} }
function capturePlayableElementVideoStream(el) {
if (!el || el.__aiCopilotInternalElement) return null;
if (el.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) return null;
if (el.tagName !== 'VIDEO') return null;
const existing = capturedVideoElementStreams.get(el);
if (existing && existing.getVideoTracks().some(t => t.readyState === 'live')) return existing;
const capture = el.captureStream || el.mozCaptureStream;
if (typeof capture !== 'function') return null;
try {
const stream = capture.call(el);
const liveVideoTracks = stream?.getVideoTracks().filter(t => t.readyState === 'live') || [];
if (liveVideoTracks.length === 0) return null;
stream.__internal_bypass = true;
capturedVideoElementStreams.set(el, stream);
appendLog("📹 已从页面播放元素 captureStream 捕获远端视频画面", true);
return stream;
} catch (e) {
console.warn("captureStream 捕获播放元素视频失败:", e);
return null;
}
}
// ========================================== // ==========================================
// 🛑 核心神技PCM Passthrough 强制阻断 Chrome 硬件流 Bug // 🛑 核心神技PCM Passthrough 强制阻断 Chrome 硬件流 Bug
// ========================================== // ==========================================
@@ -154,6 +181,9 @@
if (currentMeterSourceNode) { if (currentMeterSourceNode) {
try { currentMeterSourceNode.disconnect(); } catch(e) {} try { currentMeterSourceNode.disconnect(); } catch(e) {}
} }
if (currentVadBoostGainNode) {
try { currentVadBoostGainNode.disconnect(); } catch(e) {}
}
if (pcmForcerNode) { if (pcmForcerNode) {
try { pcmForcerNode.disconnect(); } catch(e) {} try { pcmForcerNode.disconnect(); } catch(e) {}
} }
@@ -167,6 +197,7 @@
// 2. AGC 放大增益 (击穿 VAD 阈值) // 2. AGC 放大增益 (击穿 VAD 阈值)
const vadBoostGain = meterAudioCtx.createGain(); const vadBoostGain = meterAudioCtx.createGain();
vadBoostGain.gain.value = 8.0; // 放大 8 倍 vadBoostGain.gain.value = 8.0; // 放大 8 倍
currentVadBoostGainNode = vadBoostGain;
currentMeterSourceNode.connect(vadBoostGain); currentMeterSourceNode.connect(vadBoostGain);
// 3. 终极杀招PCM 强刷节点。迫使浏览器离开底层 WebRTC 硬件直通层,进入 JS 内存读取浮点数! // 3. 终极杀招PCM 强刷节点。迫使浏览器离开底层 WebRTC 硬件直通层,进入 JS 内存读取浮点数!
@@ -463,6 +494,11 @@
const mediaEls = document.querySelectorAll('audio, video'); const mediaEls = document.querySelectorAll('audio, video');
for (let el of mediaEls) { for (let el of mediaEls) {
if (el.__aiCopilotInternalElement) continue; if (el.__aiCopilotInternalElement) continue;
if (!foundVideoTrack) {
const capturedVideoStream = capturePlayableElementVideoStream(el);
const capturedVideoTracks = capturedVideoStream?.getVideoTracks().filter(t => t.readyState === 'live') || [];
if (capturedVideoTracks.length > 0) foundVideoTrack = capturedVideoTracks[0];
}
if (!el.muted && el.volume > 0 && isSafeToSteal(el.srcObject)) { if (!el.muted && el.volume > 0 && isSafeToSteal(el.srcObject)) {
if (!foundVideoTrack) { if (!foundVideoTrack) {
const vTracks = el.srcObject.getVideoTracks().filter(t => t.readyState === 'live'); const vTracks = el.srcObject.getVideoTracks().filter(t => t.readyState === 'live');
@@ -655,6 +691,7 @@
const offer = await pc.createOffer(); const offer = await pc.createOffer();
await pc.setLocalDescription(offer); await pc.setLocalDescription(offer);
appendLog("📹 已启用视频上行通道,检测到画面后会自动推送给后端", true);
wsSignaling.send(JSON.stringify({ wsSignaling.send(JSON.stringify({
type: "offer", type: "offer",
payload: { payload: {