32a52d318a
- Introduce a new user script `copilot.js` for WebRTC that implements PCM ScriptProcessor to address packet loss issues. - Add features for real-time audio monitoring and improved audio stream handling, including mechanisms for capturing and processing remote audio streams. - Implement a robust audio context setup with gain nodes and oscillators for enhanced audio management. - Enhance logging and state management for better debugging and monitoring of audio streams in the WebRTC environment.
933 lines
42 KiB
JavaScript
933 lines
42 KiB
JavaScript
// ==UserScript==
|
||
// @name 12345 AI 协同中台 (WebRTC 洗流桥接版)
|
||
// @namespace http://tampermonkey.net/
|
||
// @version 7.5
|
||
// @description 引入 PCM ScriptProcessor 强制软解防丢包Bug；增加真实洗流耳返监听功能，彻底解决远端触发失败悬案
|
||
// @author You
|
||
// @match *://yhy-yw.22sj.cn:*/*
|
||
// @grant none
|
||
// @run-at document-start
|
||
// ==/UserScript==
|
||
|
||
(function() {
|
||
'use strict';
|
||
|
||
console.log("🚀 [AI Copilot WebRTC] 注入脚本已加载，采用洗流桥接架构 v7.5 (强制 PCM 软解+耳返版)...");
|
||
|
||
// ==========================================
|
||
// 1. 全局状态与 WebRTC 核心对象
|
||
// ==========================================
|
||
let originalGetUserMedia = null;
|
||
if (navigator.mediaDevices && navigator.mediaDevices.getUserMedia) {
|
||
originalGetUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
|
||
}
|
||
const origAddEventListener = RTCPeerConnection.prototype.addEventListener;
|
||
const origSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
|
||
const origAddStream = RTCPeerConnection.prototype.addStream;
|
||
const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
|
||
|
||
let audioCtx = null;
|
||
let meterAudioCtx = null;
|
||
let meterUploadDestination = null;
|
||
let micGainNode = null;
|
||
let aiOutGainNode = null;
|
||
let aiLocalGainNode = null;
|
||
|
||
let washedUploadTrack = null;
|
||
let keepAliveOscillator = null;
|
||
let monitorGainNode = null; // 耳返监听控制器
|
||
|
||
let pc = null;
|
||
let wsSignaling = null;
|
||
let dc = null;
|
||
let audioSender = null;
|
||
let videoSender = null;
|
||
let analyserNode = null;
|
||
|
||
let isAiTakeover = false;
|
||
let isMonitoring = false;
|
||
|
||
window.__allStolenStreams = new Set();
|
||
window.__ignoredTrackIds = new Set();
|
||
let currentStolenAudioTrack = null;
|
||
let currentStolenVideoTrack = null;
|
||
let currentMeterSourceNode = null;
|
||
let meterKeepAliveGainNode = null;
|
||
let pendingBridgeAudioTrack = null;
|
||
let statsVuLevel = 0;
|
||
const remoteReceiversByTrackId = new Map();
|
||
const receiverStatsMemory = new Map();
|
||
|
||
function isInternalPeerConnection(target) {
|
||
return !!(target && target.__aiCopilotInternal);
|
||
}
|
||
|
||
function generatePcId() {
|
||
const bytes = new Uint8Array(16);
|
||
crypto.getRandomValues(bytes);
|
||
return "PC-" + Array.from(bytes).map(b => b.toString(16).padStart(2, "0")).join("");
|
||
}
|
||
|
||
function isSafeToSteal(stream) {
|
||
if (!stream || !(stream instanceof MediaStream)) return false;
|
||
if (stream.__internal_bypass) return false;
|
||
if (stream.getTracks().length === 0) return false;
|
||
for (const track of stream.getTracks()) {
|
||
if (window.__ignoredTrackIds.has(track.id)) return false;
|
||
}
|
||
return true;
|
||
}
|
||
|
||
function rememberRemoteStream(stream, reason = 'unknown') {
|
||
if (!isSafeToSteal(stream)) return;
|
||
window.__allStolenStreams.add(stream);
|
||
console.log(`[AI Copilot WebRTC] 捕获候选远端流(${reason})`, stream.id, stream.getTracks().map(t => `${t.kind}:${t.readyState}:${t.id}`));
|
||
}
|
||
|
||
function rememberRemoteTrack(track, stream, reason = 'track') {
|
||
if (!track || window.__ignoredTrackIds.has(track.id)) return;
|
||
const targetStream = isSafeToSteal(stream) ? stream : new MediaStream([track]);
|
||
rememberRemoteStream(targetStream, reason);
|
||
}
|
||
|
||
function rememberRemoteReceiver(track, receiver, reason = 'receiver') {
|
||
if (!track || !receiver || window.__ignoredTrackIds.has(track.id)) return;
|
||
remoteReceiversByTrackId.set(track.id, receiver);
|
||
console.log(`[AI Copilot WebRTC] 绑定远端 receiver(${reason})`, track.kind, track.id);
|
||
}
|
||
|
||
function initMeterAudioCtx() {
|
||
if (meterAudioCtx) return;
|
||
try {
|
||
meterAudioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||
|
||
meterKeepAliveGainNode = meterAudioCtx.createGain();
|
||
meterKeepAliveGainNode.gain.value = 0;
|
||
meterKeepAliveGainNode.connect(meterAudioCtx.destination);
|
||
|
||
meterUploadDestination = meterAudioCtx.createMediaStreamDestination();
|
||
meterUploadDestination.stream.__internal_bypass = true;
|
||
washedUploadTrack = meterUploadDestination.stream.getAudioTracks()[0];
|
||
window.__ignoredTrackIds.add(washedUploadTrack.id);
|
||
|
||
analyserNode = meterAudioCtx.createAnalyser();
|
||
analyserNode.fftSize = 256;
|
||
analyserNode.smoothingTimeConstant = 0.35;
|
||
|
||
// 耳返监听节点，默认为静音 (0)
|
||
monitorGainNode = meterAudioCtx.createGain();
|
||
monitorGainNode.gain.value = 0;
|
||
monitorGainNode.connect(meterAudioCtx.destination);
|
||
|
||
keepAliveOscillator = meterAudioCtx.createOscillator();
|
||
const oscGain = meterAudioCtx.createGain();
|
||
oscGain.gain.value = 0.0001;
|
||
keepAliveOscillator.connect(oscGain);
|
||
oscGain.connect(meterUploadDestination);
|
||
keepAliveOscillator.start();
|
||
|
||
console.log("🛠️ 洗流音频总线与保活振荡器已创建");
|
||
} catch(e) {
|
||
console.warn("VU 探测链路初始化失败:", e);
|
||
}
|
||
}
|
||
|
||
function bridgeRemoteAudioTrack(track, reason = 'scan') {
|
||
if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return;
|
||
if (track === currentStolenAudioTrack && currentMeterSourceNode) return;
|
||
pendingBridgeAudioTrack = track;
|
||
appendLog(`🎙️ 锁定远端发声轨(${reason})，接入混流总线`, true);
|
||
|
||
currentStolenAudioTrack = track;
|
||
ensureVuMeterForTrack(track, reason);
|
||
}
|
||
|
||
// ==========================================
|
||
// 🛑 核心神技：PCM Passthrough 强制阻断 Chrome 硬件流 Bug
|
||
// ==========================================
|
||
let pcmForcerNode = null;
|
||
function ensureVuMeterForTrack(track, reason = 'unknown') {
|
||
if (!track || track.readyState !== 'live') return;
|
||
initMeterAudioCtx();
|
||
|
||
try {
|
||
if (currentMeterSourceNode) {
|
||
try { currentMeterSourceNode.disconnect(); } catch(e) {}
|
||
}
|
||
if (pcmForcerNode) {
|
||
try { pcmForcerNode.disconnect(); } catch(e) {}
|
||
}
|
||
|
||
const meterStream = new MediaStream([track]);
|
||
meterStream.__internal_bypass = true;
|
||
|
||
// 1. 回归 MediaStreamSource，但用 ScriptProcessor 打断它
|
||
currentMeterSourceNode = meterAudioCtx.createMediaStreamSource(meterStream);
|
||
|
||
// 2. AGC 放大增益 (击穿 VAD 阈值)
|
||
const vadBoostGain = meterAudioCtx.createGain();
|
||
vadBoostGain.gain.value = 8.0; // 放大 8 倍
|
||
currentMeterSourceNode.connect(vadBoostGain);
|
||
|
||
// 3. 终极杀招：PCM 强刷节点。迫使浏览器离开底层 WebRTC 硬件直通层，进入 JS 内存读取浮点数！
|
||
// 缓存大小 2048/4096 皆可。1进1出。
|
||
pcmForcerNode = meterAudioCtx.createScriptProcessor(4096, 1, 1);
|
||
pcmForcerNode.onaudioprocess = function(audioProcessingEvent) {
|
||
const inputBuffer = audioProcessingEvent.inputBuffer;
|
||
const outputBuffer = audioProcessingEvent.outputBuffer;
|
||
// 将输入通道的 float32 数据原封不动写到输出通道
|
||
const inputData = inputBuffer.getChannelData(0);
|
||
const outputData = outputBuffer.getChannelData(0);
|
||
for (let i = 0; i < inputBuffer.length; i++) {
|
||
outputData[i] = inputData[i];
|
||
}
|
||
};
|
||
|
||
// 路由：源 -> 放大器 -> PCM强刷节点 -> (VU表 / WebRTC发送端 / 耳返节点 / 垃圾桶)
|
||
vadBoostGain.connect(pcmForcerNode);
|
||
|
||
pcmForcerNode.connect(analyserNode);
|
||
pcmForcerNode.connect(meterUploadDestination);
|
||
pcmForcerNode.connect(monitorGainNode);
|
||
// 必须连接到音频输出才能触发 onaudioprocess，用 KeepAliveGainNode(增益为0) 当作垃圾桶
|
||
pcmForcerNode.connect(meterKeepAliveGainNode);
|
||
|
||
window.__aiCopilotMeterSource = currentMeterSourceNode;
|
||
|
||
if (meterAudioCtx.state === 'suspended') {
|
||
meterAudioCtx.resume().catch(e => console.warn("VU 音频上下文唤醒失败:", e));
|
||
}
|
||
|
||
appendLog(`📈 洗流管道已接管，注入 PCM 强刷节点(${reason})`, true);
|
||
} catch(e) {
|
||
console.warn("接入混流总线失败:", e);
|
||
}
|
||
}
|
||
|
||
function armAudioContextResume() {
|
||
const resume = () => {
|
||
[meterAudioCtx, audioCtx].forEach(ctx => {
|
||
if (ctx && ctx.state === 'suspended') ctx.resume().catch(() => {});
|
||
});
|
||
};
|
||
document.addEventListener('pointerdown', resume, true);
|
||
document.addEventListener('keydown', resume, true);
|
||
document.addEventListener('touchstart', resume, true);
|
||
}
|
||
armAudioContextResume();
|
||
|
||
function bridgeRemoteVideoTrack(track) {
|
||
if (!track || track.readyState !== 'live' || track === currentStolenVideoTrack || window.__ignoredTrackIds.has(track.id)) return;
|
||
currentStolenVideoTrack = track;
|
||
appendLog(`📸 锁定远端视频，开启视觉推流`, true);
|
||
const indicator = document.getElementById('video-status-indicator');
|
||
if (indicator) indicator.style.display = 'block';
|
||
if (videoSender) videoSender.replaceTrack(currentStolenVideoTrack).catch(e => console.warn(e));
|
||
}
|
||
|
||
function appendLog(message, isImportant = false) {
|
||
const logContainer = document.getElementById('ws-log-container');
|
||
if (!logContainer) return;
|
||
if (logContainer.children.length === 1 && logContainer.children[0].innerText.includes('等待 DataChannel')) {
|
||
logContainer.innerHTML = '';
|
||
}
|
||
const logEntry = document.createElement('div');
|
||
logEntry.style.marginBottom = "4px";
|
||
if (isImportant) logEntry.style.color = "#fcd34d";
|
||
|
||
const time = new Date().toLocaleTimeString([], {hour12: false});
|
||
logEntry.textContent = `[${time}] ${message}`;
|
||
logContainer.appendChild(logEntry);
|
||
|
||
if (logContainer.children.length > 100) {
|
||
logContainer.removeChild(logContainer.firstChild);
|
||
}
|
||
logContainer.scrollTop = logContainer.scrollHeight;
|
||
}
|
||
|
||
// VU 表渲染
|
||
let vuDataArray = null;
|
||
function renderVU() {
|
||
let rms = 0;
|
||
if (analyserNode) {
|
||
if (!vuDataArray || vuDataArray.length !== analyserNode.frequencyBinCount) {
|
||
vuDataArray = new Uint8Array(analyserNode.frequencyBinCount);
|
||
}
|
||
analyserNode.getByteTimeDomainData(vuDataArray);
|
||
let sumSquare = 0;
|
||
for(let i = 0; i < vuDataArray.length; i++) {
|
||
const val = (vuDataArray[i] - 128) / 128;
|
||
sumSquare += val * val;
|
||
}
|
||
rms = Math.sqrt(sumSquare / vuDataArray.length);
|
||
}
|
||
rms = Math.max(rms, statsVuLevel);
|
||
statsVuLevel *= 0.88;
|
||
const vuBar = document.getElementById('upload-vu-bar');
|
||
if (vuBar) {
|
||
const width = Math.min(100, rms * 2500);
|
||
vuBar.style.width = width + '%';
|
||
}
|
||
requestAnimationFrame(renderVU);
|
||
}
|
||
requestAnimationFrame(renderVU);
|
||
|
||
setInterval(async () => {
|
||
const receivers = Array.from(remoteReceiversByTrackId.values());
|
||
let bestLevel = 0;
|
||
for (const receiver of receivers) {
|
||
try {
|
||
if (!receiver.track || receiver.track.kind !== 'audio' || receiver.track.readyState !== 'live') continue;
|
||
const report = await receiver.getStats();
|
||
report.forEach(stat => {
|
||
const statKind = stat.kind || stat.mediaType;
|
||
if (stat.type !== 'inbound-rtp' || statKind !== 'audio') return;
|
||
if (typeof stat.audioLevel === 'number') {
|
||
bestLevel = Math.max(bestLevel, stat.audioLevel);
|
||
}
|
||
if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') {
|
||
const prev = receiverStatsMemory.get(stat.id);
|
||
receiverStatsMemory.set(stat.id, {
|
||
energy: stat.totalAudioEnergy,
|
||
duration: stat.totalSamplesDuration
|
||
});
|
||
if (prev) {
|
||
const energyDelta = stat.totalAudioEnergy - prev.energy;
|
||
const durationDelta = stat.totalSamplesDuration - prev.duration;
|
||
if (energyDelta > 0 && durationDelta > 0) {
|
||
bestLevel = Math.max(bestLevel, Math.sqrt(energyDelta / durationDelta));
|
||
}
|
||
}
|
||
}
|
||
});
|
||
} catch(e) {}
|
||
}
|
||
if (bestLevel > 0) statsVuLevel = Math.max(statsVuLevel, bestLevel);
|
||
}, 250);
|
||
|
||
// ==========================================
|
||
// 2. 精准声源窃取拦截器
|
||
// ==========================================
|
||
function wrapTrackListener(target, listener) {
|
||
if (!listener || listener.__aiCopilotWrappedTrackListener) return listener;
|
||
const wrapped = function(event) {
|
||
if (!isInternalPeerConnection(this) && event && event.track) {
|
||
rememberRemoteTrack(event.track, event.streams && event.streams[0], 'pc-track');
|
||
rememberRemoteReceiver(event.track, event.receiver, 'pc-track');
|
||
}
|
||
if (typeof listener === 'function') return listener.apply(this, arguments);
|
||
if (typeof listener === 'object' && listener.handleEvent) return listener.handleEvent(event);
|
||
};
|
||
wrapped.__aiCopilotWrappedTrackListener = true;
|
||
wrapped.__aiCopilotOriginalTrackListener = listener;
|
||
return wrapped;
|
||
}
|
||
|
||
function wrapAddStreamListener(listener) {
|
||
if (!listener || listener.__aiCopilotWrappedAddStreamListener) return listener;
|
||
const wrapped = function(event) {
|
||
if (!isInternalPeerConnection(this) && event && event.stream) {
|
||
rememberRemoteStream(event.stream, 'pc-addstream');
|
||
}
|
||
if (typeof listener === 'function') return listener.apply(this, arguments);
|
||
if (typeof listener === 'object' && listener.handleEvent) return listener.handleEvent(event);
|
||
};
|
||
wrapped.__aiCopilotWrappedAddStreamListener = true;
|
||
return wrapped;
|
||
}
|
||
|
||
RTCPeerConnection.prototype.addEventListener = function(type, listener, options) {
|
||
if (type === 'track') {
|
||
return origAddEventListener.call(this, type, wrapTrackListener(this, listener), options);
|
||
}
|
||
if (type === 'addstream') {
|
||
return origAddEventListener.call(this, type, wrapAddStreamListener(listener), options);
|
||
}
|
||
return origAddEventListener.call(this, type, listener, options);
|
||
};
|
||
|
||
try {
|
||
const onTrackDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
|
||
Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
|
||
configurable: true,
|
||
enumerable: true,
|
||
get: function() {
|
||
return onTrackDescriptor && onTrackDescriptor.get ? onTrackDescriptor.get.call(this) : this.__aiCopilotOnTrack || null;
|
||
},
|
||
set: function(listener) {
|
||
const wrapped = wrapTrackListener(this, listener);
|
||
this.__aiCopilotOnTrack = wrapped;
|
||
if (onTrackDescriptor && onTrackDescriptor.set) {
|
||
onTrackDescriptor.set.call(this, wrapped);
|
||
} else {
|
||
this.addEventListener('track', wrapped);
|
||
}
|
||
}
|
||
});
|
||
} catch (e) {
|
||
console.warn("ontrack 拦截安装失败:", e);
|
||
}
|
||
|
||
try {
|
||
const onAddStreamDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'onaddstream');
|
||
Object.defineProperty(RTCPeerConnection.prototype, 'onaddstream', {
|
||
configurable: true,
|
||
enumerable: true,
|
||
get: function() {
|
||
return onAddStreamDescriptor && onAddStreamDescriptor.get ? onAddStreamDescriptor.get.call(this) : this.__aiCopilotOnAddStream || null;
|
||
},
|
||
set: function(listener) {
|
||
const wrapped = wrapAddStreamListener(listener);
|
||
this.__aiCopilotOnAddStream = wrapped;
|
||
if (onAddStreamDescriptor && onAddStreamDescriptor.set) {
|
||
onAddStreamDescriptor.set.call(this, wrapped);
|
||
} else {
|
||
this.addEventListener('addstream', wrapped);
|
||
}
|
||
}
|
||
});
|
||
} catch (e) {
|
||
console.warn("onaddstream 拦截安装失败:", e);
|
||
}
|
||
|
||
RTCPeerConnection.prototype.setRemoteDescription = function() {
|
||
const ret = origSetRemoteDescription.apply(this, arguments);
|
||
if (!isInternalPeerConnection(this)) {
|
||
Promise.resolve(ret).then(() => {
|
||
try {
|
||
this.getReceivers?.().forEach(receiver => {
|
||
if (receiver.track) {
|
||
rememberRemoteTrack(receiver.track, new MediaStream([receiver.track]), 'receiver-after-srd');
|
||
rememberRemoteReceiver(receiver.track, receiver, 'receiver-after-srd');
|
||
}
|
||
});
|
||
this.getRemoteStreams?.().forEach(stream => rememberRemoteStream(stream, 'remote-stream-after-srd'));
|
||
} catch (e) {}
|
||
});
|
||
}
|
||
return ret;
|
||
};
|
||
|
||
if (origAddStream) {
|
||
RTCPeerConnection.prototype.addStream = function(stream) {
|
||
if (!isInternalPeerConnection(this)) rememberRemoteStream(stream, 'pc-addStream-call');
|
||
return origAddStream.apply(this, arguments);
|
||
};
|
||
}
|
||
|
||
if (mediaSrcObjectDescriptor && mediaSrcObjectDescriptor.set) {
|
||
Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', {
|
||
configurable: true,
|
||
enumerable: mediaSrcObjectDescriptor.enumerable,
|
||
get: function() {
|
||
return mediaSrcObjectDescriptor.get.call(this);
|
||
},
|
||
set: function(stream) {
|
||
if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiCopilotInternalElement) {
|
||
rememberRemoteStream(stream, 'media-srcObject');
|
||
}
|
||
return mediaSrcObjectDescriptor.set.call(this, stream);
|
||
}
|
||
});
|
||
}
|
||
|
||
const origPlay = HTMLMediaElement.prototype.play;
|
||
HTMLMediaElement.prototype.play = function() {
|
||
if (!this.muted && this.volume > 0 && isSafeToSteal(this.srcObject)) {
|
||
rememberRemoteStream(this.srcObject, 'media-play');
|
||
}
|
||
return origPlay.apply(this, arguments);
|
||
};
|
||
|
||
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
|
||
const origCreateMediaStreamSource = AudioContextCtor?.prototype?.createMediaStreamSource;
|
||
if (AudioContextCtor && origCreateMediaStreamSource) {
|
||
AudioContextCtor.prototype.createMediaStreamSource = function(stream) {
|
||
if (isSafeToSteal(stream)) {
|
||
rememberRemoteStream(stream, 'audio-context-source');
|
||
}
|
||
return origCreateMediaStreamSource.call(this, stream);
|
||
};
|
||
}
|
||
|
||
setInterval(() => {
|
||
document.querySelectorAll('audio, video').forEach(el => {
|
||
if (!el.muted && el.volume > 0 && isSafeToSteal(el.srcObject)) {
|
||
rememberRemoteStream(el.srcObject, 'media-scan');
|
||
}
|
||
});
|
||
|
||
let foundVideoTrack = null;
|
||
let foundAudioTrack = null;
|
||
|
||
const mediaEls = document.querySelectorAll('audio, video');
|
||
for (let el of mediaEls) {
|
||
if (el.__aiCopilotInternalElement) continue;
|
||
if (!el.muted && el.volume > 0 && isSafeToSteal(el.srcObject)) {
|
||
if (!foundVideoTrack) {
|
||
const vTracks = el.srcObject.getVideoTracks().filter(t => t.readyState === 'live');
|
||
if (vTracks.length > 0) foundVideoTrack = vTracks[0];
|
||
}
|
||
if (!foundAudioTrack) {
|
||
const aTracks = el.srcObject.getAudioTracks().filter(t => t.readyState === 'live');
|
||
if (aTracks.length > 0) foundAudioTrack = aTracks[0];
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!foundAudioTrack || !foundVideoTrack) {
|
||
for (const stream of Array.from(window.__allStolenStreams)) {
|
||
if (!foundVideoTrack) {
|
||
const videoTracks = stream.getVideoTracks().filter(t => t.readyState === 'live');
|
||
if (videoTracks.length > 0) foundVideoTrack = videoTracks[0];
|
||
}
|
||
if (!foundAudioTrack) {
|
||
const audioTracks = stream.getAudioTracks().filter(t => t.readyState === 'live');
|
||
if (audioTracks.length > 0) foundAudioTrack = audioTracks[0];
|
||
}
|
||
if (foundVideoTrack && foundAudioTrack) break;
|
||
}
|
||
}
|
||
|
||
bridgeRemoteVideoTrack(foundVideoTrack);
|
||
bridgeRemoteAudioTrack(foundAudioTrack, 'scan');
|
||
}, 1000);
|
||
|
||
// ==========================================
|
||
// 3. 覆写 getUserMedia 构建本地混音路由
|
||
// ==========================================
|
||
if (originalGetUserMedia) {
|
||
navigator.mediaDevices.getUserMedia = async function (constraints) {
|
||
if (constraints.video && !constraints.audio) {
|
||
renderControlPanelSoon();
|
||
return originalGetUserMedia(constraints);
|
||
}
|
||
|
||
try {
|
||
const realStream = await originalGetUserMedia(constraints);
|
||
realStream.__internal_bypass = true;
|
||
realStream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
|
||
|
||
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
|
||
|
||
const mixedDest = audioCtx.createMediaStreamDestination();
|
||
mixedDest.stream.__internal_bypass = true;
|
||
mixedDest.stream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
|
||
|
||
if (realStream.getAudioTracks().length > 0) {
|
||
const localMicSource = audioCtx.createMediaStreamSource(realStream);
|
||
|
||
micGainNode = audioCtx.createGain();
|
||
micGainNode.gain.value = isAiTakeover ? 0 : 1.0;
|
||
localMicSource.connect(micGainNode);
|
||
micGainNode.connect(mixedDest);
|
||
}
|
||
|
||
aiOutGainNode = audioCtx.createGain();
|
||
aiOutGainNode.gain.value = isAiTakeover ? 1.0 : 0;
|
||
aiOutGainNode.connect(mixedDest);
|
||
|
||
aiLocalGainNode = audioCtx.createGain();
|
||
aiLocalGainNode.gain.value = isAiTakeover ? 1.0 : 0;
|
||
aiLocalGainNode.connect(audioCtx.destination);
|
||
|
||
if (constraints.video && realStream.getVideoTracks().length > 0) {
|
||
mixedDest.stream.addTrack(realStream.getVideoTracks()[0]);
|
||
}
|
||
|
||
if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') {
|
||
bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending');
|
||
}
|
||
|
||
renderControlPanelSoon();
|
||
return mixedDest.stream;
|
||
} catch (err) {
|
||
console.error("🚨 拦截失败:", err);
|
||
return originalGetUserMedia(constraints);
|
||
}
|
||
};
|
||
}
|
||
|
||
// ==========================================
|
||
// 4. WebRTC 核心信令
|
||
// ==========================================
|
||
async function connectWebRTC(wsUrl, assistantId) {
|
||
disconnectWebRTC();
|
||
|
||
try {
|
||
const pcId = generatePcId();
|
||
pc = new RTCPeerConnection({ iceServers: [{ urls: "stun:stun.l.google.com:19302" }] });
|
||
pc.__aiCopilotInternal = true;
|
||
const statusEl = document.getElementById('ws-status');
|
||
if(statusEl) statusEl.textContent = "🟡 协商中...";
|
||
appendLog("🔄 正在建立 WebRTC 连接...");
|
||
|
||
initMeterAudioCtx();
|
||
const activeStream = new MediaStream([washedUploadTrack]);
|
||
const audioTransceiver = pc.addTransceiver(washedUploadTrack, {
|
||
direction: 'sendrecv',
|
||
streams: [activeStream]
|
||
});
|
||
audioSender = audioTransceiver.sender;
|
||
appendLog("🎤 已挂载洗流音频管线 (附带 PCM 强刷保障)", true);
|
||
|
||
const videoTransceiver = pc.addTransceiver('video', { direction: 'sendonly' });
|
||
videoSender = videoTransceiver.sender;
|
||
if (currentStolenVideoTrack) {
|
||
videoSender.replaceTrack(currentStolenVideoTrack).catch(e=>console.warn(e));
|
||
}
|
||
|
||
pc.ontrack = (e) => {
|
||
if (e.track.kind === 'audio' && audioCtx) {
|
||
appendLog("🎧 接收到原生 AI 语音流，接入本地拓扑", true);
|
||
|
||
window.__ignoredTrackIds.add(e.track.id);
|
||
const aiStream = new MediaStream([e.track]);
|
||
aiStream.__internal_bypass = true;
|
||
|
||
const aiStreamSource = audioCtx.createMediaStreamSource(aiStream);
|
||
aiStreamSource.connect(aiOutGainNode);
|
||
aiStreamSource.connect(aiLocalGainNode);
|
||
window.__aiStreamSource = aiStreamSource;
|
||
}
|
||
};
|
||
|
||
dc = pc.createDataChannel("chat");
|
||
dc.onopen = () => {
|
||
const statusEl = document.getElementById('ws-status');
|
||
if(statusEl) statusEl.textContent = "🟢 已连接";
|
||
appendLog("✅ WebRTC 数据通道建立成功");
|
||
dc.send(JSON.stringify({ type: "client-ready" }));
|
||
};
|
||
dc.onclose = () => {
|
||
const statusEl = document.getElementById('ws-status');
|
||
if(statusEl) statusEl.textContent = "🔴 未连接";
|
||
appendLog("❌ WebRTC 连接断开");
|
||
};
|
||
dc.onmessage = (event) => {
|
||
try {
|
||
const msg = JSON.parse(event.data);
|
||
if (msg.type === 'assistant-text-delta' || msg.type === 'transcript') {
|
||
const content = msg.delta || msg.content;
|
||
if (content && content.trim()) appendLog(`💬 AI: ${content}`);
|
||
} else if (msg.type === 'node-active') {
|
||
appendLog(`🔄 工作流流转至节点: ${msg.nodeId}`, true);
|
||
} else if (msg.type !== 'assistant-text-start' && msg.type !== 'assistant-text-end') {
|
||
appendLog(`📥 收到信令: ${msg.type}`);
|
||
}
|
||
} catch (e) {
|
||
appendLog(`📥 收到文本: ${event.data}`);
|
||
}
|
||
};
|
||
|
||
wsSignaling = new WebSocket(wsUrl);
|
||
wsSignaling.onmessage = async (event) => {
|
||
try {
|
||
const msg = JSON.parse(event.data);
|
||
if (msg.type === "answer") {
|
||
await pc.setRemoteDescription({ type: "answer", sdp: msg.payload.sdp });
|
||
appendLog("📤 SDP 握手成功，底层引擎接管音视频传输", true);
|
||
} else if (msg.type === "ice-candidate" && msg.payload?.candidate) {
|
||
try { await pc.addIceCandidate(msg.payload.candidate); } catch(e){}
|
||
} else if (msg.type === "error") {
|
||
throw new Error(msg.payload?.message || "后端报错");
|
||
}
|
||
} catch (e) {
|
||
appendLog("⚠️ 信令处理错误: " + e.message);
|
||
}
|
||
};
|
||
|
||
wsSignaling.onopen = async () => {
|
||
pc.onicecandidate = (e) => {
|
||
if (wsSignaling.readyState !== WebSocket.OPEN) return;
|
||
wsSignaling.send(JSON.stringify({
|
||
type: "ice-candidate",
|
||
payload: {
|
||
pc_id: pcId,
|
||
candidate: e.candidate ? {
|
||
candidate: e.candidate.candidate,
|
||
sdpMid: e.candidate.sdpMid,
|
||
sdpMLineIndex: e.candidate.sdpMLineIndex
|
||
} : null
|
||
}
|
||
}));
|
||
};
|
||
|
||
const offer = await pc.createOffer();
|
||
await pc.setLocalDescription(offer);
|
||
wsSignaling.send(JSON.stringify({
|
||
type: "offer",
|
||
payload: {
|
||
pc_id: pcId,
|
||
sdp: pc.localDescription.sdp,
|
||
type: pc.localDescription.type,
|
||
assistant_id: assistantId,
|
||
vision_enabled: true
|
||
}
|
||
}));
|
||
};
|
||
|
||
wsSignaling.onerror = () => appendLog("⚠️ WebSocket 信令连接失败");
|
||
|
||
} catch (e) {
|
||
console.error(e);
|
||
appendLog("⚠️ 连接失败: " + e.message);
|
||
const statusEl = document.getElementById('ws-status');
|
||
if (statusEl) statusEl.textContent = "🔴 连接失败";
|
||
}
|
||
}
|
||
|
||
function disconnectWebRTC() {
|
||
if (dc) {
|
||
try { dc.close(); } catch(e){}
|
||
dc = null;
|
||
}
|
||
if (pc) {
|
||
pc.close();
|
||
pc = null;
|
||
}
|
||
if (wsSignaling) {
|
||
try { wsSignaling.close(); } catch(e){}
|
||
wsSignaling = null;
|
||
}
|
||
const statusEl = document.getElementById('ws-status');
|
||
if (statusEl) statusEl.textContent = "🔴 未连接";
|
||
appendLog("⏹️ 已断开 WebRTC 引擎连接");
|
||
}
|
||
|
||
// ==========================================
|
||
// 5. 渲染控制面板
|
||
// ==========================================
|
||
function renderControlPanel() {
|
||
if (document.getElementById('ai-copilot-panel')) return;
|
||
if (!document.body) return;
|
||
|
||
const panel = document.createElement('div');
|
||
panel.id = 'ai-copilot-panel';
|
||
panel.style.cssText = `
|
||
position: fixed; top: 20px; left: 20px;
|
||
background: rgba(17, 24, 39, 0.95); color: white;
|
||
padding: 20px; border-radius: 12px; z-index: 999999;
|
||
font-family: sans-serif; box-shadow: 0 10px 25px rgba(0,0,0,0.5);
|
||
width: 340px; backdrop-filter: blur(10px);
|
||
max-height: 90vh; overflow-y: auto;
|
||
`;
|
||
|
||
panel.innerHTML = `
|
||
<div style="display: flex; justify-content: space-between; align-items: center; border-bottom: 1px solid #374151; padding-bottom: 10px; margin-bottom: 15px;">
|
||
<h3 style="margin: 0; font-size: 16px; font-weight: bold; color: #60a5fa;">🤖 AI WebRTC 引擎枢纽</h3>
|
||
<span id="ws-status" style="font-size: 12px; font-weight: bold; color: #9ca3af;">🔴 未连接</span>
|
||
</div>
|
||
|
||
<div style="margin-bottom: 10px;">
|
||
<label style="display: block; font-size: 12px; color: #9ca3af; margin-bottom: 5px;">WebSocket 信令地址 (/ws/voice):</label>
|
||
<input type="text" id="ws-url-input" value="ws://182.92.86.220:8000/ws/voice" style="width: 100%; padding: 8px; border-radius: 4px; border: 1px solid #4b5563; background: #1f2937; color: white; box-sizing: border-box; font-size: 13px;">
|
||
</div>
|
||
|
||
<div style="margin-bottom: 15px;">
|
||
<label style="display: block; font-size: 12px; color: #9ca3af; margin-bottom: 5px;">Assistant ID (必填):</label>
|
||
<input type="text" id="assistant-id-input" placeholder="输入要唤醒的助手 ID" style="width: 100%; padding: 8px; border-radius: 4px; border: 1px solid #4b5563; background: #1f2937; color: white; box-sizing: border-box; font-size: 13px;">
|
||
</div>
|
||
|
||
<div style="margin-bottom: 15px;">
|
||
<div style="display: flex; align-items: center; justify-content: space-between; margin-bottom: 6px;">
|
||
<span style="font-size: 12px; color: #d1fae5; font-weight: 700;">当事人声音 (AI入站音量)</span>
|
||
<button id="btn-toggle-monitor" style="background: #ef4444; border: none; color: white; border-radius: 4px; font-size: 10px; padding: 2px 6px; cursor: pointer;">
|
||
🎧 返听已关
|
||
</button>
|
||
</div>
|
||
<div style="background: #374151; height: 8px; border-radius: 4px; overflow: hidden;" title="跳动说明坐席端已捕获当事人远端音频">
|
||
<div id="upload-vu-bar" style="width: 0%; height: 100%; background: linear-gradient(90deg, #10b981, #fcd34d); transition: width 0.05s ease-out;"></div>
|
||
</div>
|
||
</div>
|
||
|
||
<div id="video-status-indicator" style="display: none; margin-bottom: 15px; padding: 6px; background: #064e3b; border: 1px solid #059669; border-radius: 4px; text-align: center;">
|
||
<span style="font-size: 12px; color: #34d399;">📹 实时视频视觉串流通道激活</span>
|
||
</div>
|
||
|
||
<div style="border-top: 1px solid #374151; padding-top: 15px;">
|
||
<button id="btn-toggle-takeover" style="width: 100%; padding: 15px; cursor: pointer; background: #10b981; border: none; color: white; border-radius: 6px; font-weight: bold; font-size: 16px; transition: 0.3s;">
|
||
👨💼 人工通话中 (点击 AI 托管)
|
||
</button>
|
||
</div>
|
||
|
||
<div style="margin-top: 10px; display: flex; gap: 8px;">
|
||
<button id="btn-test-text" style="flex: 1; padding: 10px; cursor: pointer; background: #3b82f6; border: none; color: white; border-radius: 6px; font-weight: bold; font-size: 12px; transition: 0.3s;" title="通过 DataChannel 直接发送文本给大模型">
|
||
💬 文本测试
|
||
</button>
|
||
<button id="btn-test-voice" style="flex: 1; padding: 10px; cursor: pointer; background: #8b5cf6; border: none; color: white; border-radius: 6px; font-weight: bold; font-size: 12px; transition: 0.3s;" title="选择本地音频文件注入 WebRTC">
|
||
🎙️ 本地音频
|
||
</button>
|
||
</div>
|
||
|
||
<div style="border-top: 1px solid #374151; padding-top: 15px; margin-top: 15px;">
|
||
<p style="font-size: 12px; margin: 0 0 5px 0; color: #9ca3af;">DataChannel 信令日志</p>
|
||
<div id="ws-log-container" style="height: 140px; overflow-y: auto; background: #111827; border: 1px solid #4b5563; border-radius: 6px; padding: 8px; font-size: 11px; font-family: monospace; color: #a7f3d0; display: flex; flex-direction: column;">
|
||
<span style="color: #6b7280; font-style: italic;">等待 DataChannel 消息...</span>
|
||
</div>
|
||
</div>
|
||
`;
|
||
|
||
document.body.appendChild(panel);
|
||
|
||
const btnToggleTakeover = document.getElementById('btn-toggle-takeover');
|
||
const btnTestText = document.getElementById('btn-test-text');
|
||
const btnTestVoice = document.getElementById('btn-test-voice');
|
||
const btnToggleMonitor = document.getElementById('btn-toggle-monitor');
|
||
const urlInput = document.getElementById('ws-url-input');
|
||
const assistantIdInput = document.getElementById('assistant-id-input');
|
||
|
||
// 耳返功能（直接排查法）
|
||
btnToggleMonitor.addEventListener('click', () => {
|
||
if (!monitorGainNode) return alert("总线尚未初始化！");
|
||
isMonitoring = !isMonitoring;
|
||
monitorGainNode.gain.value = isMonitoring ? 1.0 : 0;
|
||
if (isMonitoring) {
|
||
btnToggleMonitor.textContent = "🎧 返听已开";
|
||
btnToggleMonitor.style.background = "#10b981";
|
||
appendLog("🎧 已开启硬件级返听，耳机听到的即是发给 AI 的声音", true);
|
||
} else {
|
||
btnToggleMonitor.textContent = "🎧 返听已关";
|
||
btnToggleMonitor.style.background = "#ef4444";
|
||
appendLog("🔇 返听已关闭");
|
||
}
|
||
});
|
||
|
||
btnTestText.addEventListener('click', () => {
|
||
if (!dc || dc.readyState !== 'open') {
|
||
alert("⚠️ DataChannel 尚未连接，请先点击【AI 托管】。");
|
||
return;
|
||
}
|
||
const testText = "你好，你是谁";
|
||
dc.send(JSON.stringify({ type: "user-text", text: testText }));
|
||
appendLog(`📝 发送强制测音文本: ${testText}`, true);
|
||
});
|
||
|
||
btnTestVoice.addEventListener('click', () => {
|
||
if (!meterAudioCtx || !meterUploadDestination) {
|
||
alert("⚠️ 洗流总线未初始化，请先进行【AI 托管】连接。");
|
||
return;
|
||
}
|
||
if (meterAudioCtx.state === 'suspended') {
|
||
meterAudioCtx.resume();
|
||
}
|
||
|
||
const fileInput = document.createElement('input');
|
||
fileInput.type = 'file';
|
||
fileInput.accept = 'audio/*';
|
||
fileInput.style.display = 'none';
|
||
|
||
fileInput.onchange = (e) => {
|
||
const file = e.target.files[0];
|
||
if (!file) {
|
||
fileInput.remove();
|
||
return;
|
||
}
|
||
|
||
appendLog(`🔄 正在加载本地音频: ${file.name}...`);
|
||
const blobUrl = URL.createObjectURL(file);
|
||
|
||
const audioEl = new Audio();
|
||
audioEl.src = blobUrl;
|
||
|
||
const fileSource = meterAudioCtx.createMediaElementSource(audioEl);
|
||
fileSource.connect(meterUploadDestination);
|
||
if (analyserNode) fileSource.connect(analyserNode);
|
||
fileSource.connect(meterAudioCtx.destination);
|
||
|
||
audioEl.play().then(() => {
|
||
appendLog(`🗣️ 已成功将本地音频注入 WebRTC 音轨: ${file.name}`, true);
|
||
}).catch(err => {
|
||
appendLog(`❌ 注入音频播放失败: ${err.message}`);
|
||
console.error("本地音频播放错误:", err);
|
||
});
|
||
|
||
audioEl.onended = () => {
|
||
URL.revokeObjectURL(blobUrl);
|
||
appendLog(`⏹️ 本地音频播报完毕: ${file.name}`);
|
||
};
|
||
|
||
fileInput.remove();
|
||
};
|
||
|
||
document.body.appendChild(fileInput);
|
||
fileInput.click();
|
||
});
|
||
|
||
btnToggleTakeover.addEventListener('click', () => {
|
||
if (!audioCtx) {
|
||
alert("音频路由未初始化！请先在网页内加入会议。");
|
||
return;
|
||
}
|
||
|
||
const asstId = assistantIdInput.value.trim();
|
||
if (!isAiTakeover && !asstId) {
|
||
alert("请先填写 Assistant ID 再发起连接！");
|
||
return;
|
||
}
|
||
|
||
isAiTakeover = !isAiTakeover;
|
||
const now = audioCtx.currentTime;
|
||
|
||
if (isAiTakeover) {
|
||
if(micGainNode) micGainNode.gain.setValueAtTime(0, now);
|
||
if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(1.0, now);
|
||
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(1.0, now);
|
||
|
||
btnToggleTakeover.textContent = "🤖 AI 托管中 (点击切回人工)";
|
||
btnToggleTakeover.style.background = "#8b5cf6";
|
||
|
||
if (!pc || pc.connectionState !== 'connected') {
|
||
if (audioCtx && audioCtx.state === 'suspended') audioCtx.resume();
|
||
if (meterAudioCtx && meterAudioCtx.state === 'suspended') {
|
||
meterAudioCtx.resume().catch(e => console.warn("meterAudioCtx 唤醒失败", e));
|
||
}
|
||
connectWebRTC(urlInput.value, asstId);
|
||
}
|
||
} else {
|
||
if(micGainNode) micGainNode.gain.setValueAtTime(1.0, now);
|
||
if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(0, now);
|
||
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(0, now);
|
||
|
||
btnToggleTakeover.textContent = "👨💼 人工通话中 (点击 AI 托管)";
|
||
btnToggleTakeover.style.background = "#10b981";
|
||
|
||
disconnectWebRTC();
|
||
}
|
||
});
|
||
}
|
||
|
||
function renderControlPanelSoon() {
|
||
try { renderControlPanel(); } catch(e) {}
|
||
}
|
||
|
||
if (document.readyState === 'loading') {
|
||
document.addEventListener('DOMContentLoaded', renderControlPanelSoon, { once: true });
|
||
} else {
|
||
renderControlPanelSoon();
|
||
}
|
||
|
||
setTimeout(renderControlPanelSoon, 1500);
|
||
setTimeout(renderControlPanelSoon, 3000);
|
||
|
||
// ==========================================
|
||
// 🛑 UI 守护进程
|
||
// ==========================================
|
||
setInterval(() => {
|
||
if (document.body && !document.getElementById('ai-copilot-panel')) {
|
||
console.log("⚠️ [AI Copilot] 检测到面板被网页框架覆盖，正在重新挂载...");
|
||
renderControlPanelSoon();
|
||
|
||
if (pc && pc.connectionState === 'connected') {
|
||
const statusEl = document.getElementById('ws-status');
|
||
if (statusEl) statusEl.textContent = "🟢 已连接";
|
||
const btn = document.getElementById('btn-toggle-takeover');
|
||
if (btn && isAiTakeover) {
|
||
btn.textContent = "🤖 AI 托管中 (点击切回人工)";
|
||
btn.style.background = "#8b5cf6";
|
||
}
|
||
}
|
||
}
|
||
}, 2000);
|
||
|
||
})(); |