Add Audio Sender Sine Tester script for WebRTC audio management

- Introduce a new user script `audio-sender-sine-tester.js` that lists current WebRTC audio senders and allows temporary sine wave transmission through selected senders.
- Implement functionality for managing audio sender candidates, tracking audio statistics, and providing real-time logging of audio activities.
- Enhance audio testing capabilities with sine wave generation and track replacement features, improving overall audio management in the WebRTC environment.
- This addition supports better monitoring and testing of audio streams in applications utilizing WebRTC.
This commit is contained in:
Xin Wang
2026-07-09 13:33:02 +08:00
parent 68931c9266
commit 03478fa557
4 changed files with 640 additions and 3309 deletions

View File

@@ -1,5 +1,5 @@
// ==UserScript==
// @name 12345 AI 协同中台 (WebRTC 洗流桥接版)
// @name 12345 AI 协同中台
// @namespace http://tampermonkey.net/
// @version 7.5
// @description 引入 PCM ScriptProcessor 强制软解防丢包Bug增加真实洗流耳返监听功能彻底解决远端触发失败悬案
@@ -23,6 +23,8 @@
}
const origAddEventListener = RTCPeerConnection.prototype.addEventListener;
const origSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
const origAddTrack = RTCPeerConnection.prototype.addTrack;
const origAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
const origAddStream = RTCPeerConnection.prototype.addStream;
const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
@@ -43,6 +45,13 @@
let audioSender = null;
let videoSender = null;
let analyserNode = null;
let businessAudioSender = null;
let businessOriginalAudioTrack = null;
let aiRemoteAudioTrack = null;
let aiTrackAttachedToBusiness = false;
let aiInputMode = 'washed';
let directInputHealthTimer = null;
let directInputTrackId = null;
let isAiTakeover = false;
let isMonitoring = false;
@@ -59,6 +68,15 @@
const remoteReceiversByTrackId = new Map();
const receiverStatsMemory = new Map();
const capturedVideoElementStreams = new WeakMap();
function createAudioContext() {
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
try {
return new AudioContextClass({ sampleRate: 48000 });
} catch (e) {
console.warn("48k AudioContext 创建失败,回退浏览器默认采样率:", e);
return new AudioContextClass();
}
}
function isInternalPeerConnection(target) {
return !!(target && target.__aiCopilotInternal);
@@ -98,10 +116,193 @@
console.log(`[AI Copilot WebRTC] 绑定远端 receiver(${reason})`, track.kind, track.id);
}
function rememberBusinessAudioSender(sender, track, reason = 'unknown') {
if (!sender) return;
const senderTrack = track || sender.track;
if (senderTrack && senderTrack.kind !== 'audio') return;
businessAudioSender = sender;
if (senderTrack && senderTrack !== aiRemoteAudioTrack) {
businessOriginalAudioTrack = senderTrack;
}
console.log(`[AI Copilot WebRTC] 绑定业务通话 audio sender(${reason})`, senderTrack?.id || 'no-track-yet');
if (isAiTakeover && aiRemoteAudioTrack) attachAiTrackToBusiness();
}
function wrapBusinessSender(sender) {
if (!sender || sender.__aiCopilotWrappedSender) return sender;
const originalReplaceTrack = sender.replaceTrack?.bind(sender);
if (!originalReplaceTrack) return sender;
sender.replaceTrack = async function(track) {
const ret = await originalReplaceTrack(track);
if (track && track.kind === 'audio' && track !== aiRemoteAudioTrack && !isAiTakeover) {
rememberBusinessAudioSender(sender, track, 'sender-replaceTrack');
}
return ret;
};
sender.__aiCopilotWrappedSender = true;
return sender;
}
async function attachAiTrackToBusiness() {
if (!isAiTakeover || !businessAudioSender || !aiRemoteAudioTrack || aiRemoteAudioTrack.readyState !== 'live') return false;
try {
await businessAudioSender.replaceTrack(aiRemoteAudioTrack);
aiTrackAttachedToBusiness = true;
appendLog("🔁 已将 AI 原生音轨直接替换到业务通话上行", true);
return true;
} catch (e) {
aiTrackAttachedToBusiness = false;
appendLog(`⚠️ AI 原生音轨替换业务 sender 失败: ${e.message || e}`);
return false;
}
}
async function restoreBusinessAudioTrack() {
if (!businessAudioSender || !aiTrackAttachedToBusiness) return;
try {
await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null);
appendLog("👨‍💼 已恢复业务通话原始音轨", true);
} catch (e) {
appendLog(`⚠️ 恢复业务音轨失败: ${e.message || e}`);
} finally {
aiTrackAttachedToBusiness = false;
}
}
async function outboundAudioBytes(sender) {
if (!sender?.getStats) return null;
try {
let total = 0;
const report = await sender.getStats();
report.forEach(stat => {
const statKind = stat.kind || stat.mediaType;
if (stat.type === 'outbound-rtp' && statKind === 'audio' && typeof stat.bytesSent === 'number') {
total += stat.bytesSent;
}
});
return total;
} catch (e) {
return null;
}
}
function aiPeerReadyForMedia() {
if (!pc || !pc.remoteDescription) return false;
return (
pc.connectionState === 'connected'
|| pc.iceConnectionState === 'connected'
|| pc.iceConnectionState === 'completed'
);
}
function aiPeerStateLabel() {
if (!pc) return 'pc=null';
return `conn=${pc.connectionState},ice=${pc.iceConnectionState},signaling=${pc.signalingState},remote=${pc.remoteDescription?.type || 'none'}`;
}
function waitForAiPeerMediaReady(timeoutMs = 10000) {
if (aiPeerReadyForMedia()) return Promise.resolve(true);
return new Promise((resolve) => {
const startedAt = Date.now();
const timer = setInterval(() => {
if (aiPeerReadyForMedia()) {
clearInterval(timer);
resolve(true);
} else if (Date.now() - startedAt >= timeoutMs) {
clearInterval(timer);
resolve(false);
}
}, 250);
});
}
function disconnectWashedInputGraph() {
if (currentMeterSourceNode) {
try { currentMeterSourceNode.disconnect(); } catch(e) {}
currentMeterSourceNode = null;
}
if (currentVadBoostGainNode) {
try { currentVadBoostGainNode.disconnect(); } catch(e) {}
currentVadBoostGainNode = null;
}
if (pcmForcerNode) {
try { pcmForcerNode.disconnect(); } catch(e) {}
}
}
async function fallbackToWashedAiInput(track, reason = 'fallback') {
if (!track || track.readyState !== 'live') return;
if (directInputHealthTimer) {
clearTimeout(directInputHealthTimer);
directInputHealthTimer = null;
}
directInputTrackId = null;
if (audioSender && washedUploadTrack && audioSender.track !== washedUploadTrack) {
try {
await audioSender.replaceTrack(washedUploadTrack);
} catch (e) {
appendLog(`⚠️ AI 输入切回洗流轨失败: ${e.message || e}`);
}
}
aiInputMode = 'washed';
appendLog(`🧯 AI 输入切回洗流链路(${reason})`, true);
ensureVuMeterForTrack(track, reason);
}
async function tryDirectAiInput(track, reason = 'direct') {
if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return false;
if (!audioSender) {
pendingBridgeAudioTrack = track;
return false;
}
if (aiInputMode === 'direct' && directInputTrackId === track.id && audioSender.track === track) return true;
try {
if (directInputHealthTimer) clearTimeout(directInputHealthTimer);
await audioSender.replaceTrack(track);
aiInputMode = 'direct';
directInputTrackId = track.id;
disconnectWashedInputGraph();
appendLog(`🚀 AI 输入直连远端原生音轨(${reason}); track=${track.id}, ${aiPeerStateLabel()}`, true);
directInputHealthTimer = setTimeout(async () => {
if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
appendLog(`🩺 等待 AI WebRTC 媒体连接后检查直连输入: ${aiPeerStateLabel()}`);
const ready = await waitForAiPeerMediaReady(10000);
if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
if (!ready) {
appendLog(`⚠️ AI WebRTC 媒体连接超时,直连输入无法验证: ${aiPeerStateLabel()}`);
await fallbackToWashedAiInput(track, 'direct-ai-pc-not-ready');
return;
}
const beforeHealthBytes = await outboundAudioBytes(audioSender);
await new Promise(resolve => setTimeout(resolve, 2500));
if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
const afterBytes = await outboundAudioBytes(audioSender);
appendLog(`🩺 直连输入发包检查: before=${beforeHealthBytes}, after=${afterBytes}, ${aiPeerStateLabel()}`);
if (
afterBytes !== null
&& (
(beforeHealthBytes !== null && afterBytes <= beforeHealthBytes)
|| (beforeHealthBytes === null && afterBytes === 0)
)
) {
await fallbackToWashedAiInput(track, 'direct-no-outbound-audio');
}
}, 500);
return true;
} catch (e) {
appendLog(`⚠️ AI 输入直连失败,改用洗流: ${e.message || e}`);
await fallbackToWashedAiInput(track, 'direct-replace-failed');
return false;
}
}
function initMeterAudioCtx() {
if (meterAudioCtx) return;
try {
meterAudioCtx = new (window.AudioContext || window.webkitAudioContext)();
meterAudioCtx = createAudioContext();
meterKeepAliveGainNode = meterAudioCtx.createGain();
meterKeepAliveGainNode.gain.value = 0;
@@ -134,14 +335,23 @@
}
}
function bridgeRemoteAudioTrack(track, reason = 'scan') {
async function bridgeRemoteAudioTrack(track, reason = 'scan') {
if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return;
if (track === currentStolenAudioTrack && currentMeterSourceNode) return;
if (
track === currentStolenAudioTrack
&& (
(aiInputMode === 'direct' && audioSender?.track === track)
|| (aiInputMode === 'washed' && currentMeterSourceNode)
)
) return;
pendingBridgeAudioTrack = track;
appendLog(`🎙️ 锁定远端发声轨(${reason})接入混流总线`, true);
appendLog(`🎙️ 锁定远端发声轨(${reason})尝试直连 AI 输入`, true);
currentStolenAudioTrack = track;
ensureVuMeterForTrack(track, reason);
const directOk = await tryDirectAiInput(track, reason);
if (!directOk && audioSender) {
await fallbackToWashedAiInput(track, `${reason}-fallback`);
}
}
function capturePlayableElementVideoStream(el) {
@@ -196,7 +406,7 @@
// 2. AGC 放大增益 (击穿 VAD 阈值)
const vadBoostGain = meterAudioCtx.createGain();
vadBoostGain.gain.value = 8.0; // 放大 8 倍
vadBoostGain.gain.value = 2.0; // fallback 洗流仅轻度放大,避免底噪/回声被过度抬高
currentVadBoostGainNode = vadBoostGain;
currentMeterSourceNode.connect(vadBoostGain);
@@ -377,6 +587,28 @@
return origAddEventListener.call(this, type, listener, options);
};
RTCPeerConnection.prototype.addTrack = function(track, ...streams) {
const sender = origAddTrack.call(this, track, ...streams);
if (!isInternalPeerConnection(this) && track?.kind === 'audio') {
rememberBusinessAudioSender(wrapBusinessSender(sender), track, 'pc-addTrack');
}
return sender;
};
if (origAddTransceiver) {
RTCPeerConnection.prototype.addTransceiver = function(trackOrKind, init) {
const transceiver = origAddTransceiver.call(this, trackOrKind, init);
if (!isInternalPeerConnection(this)) {
const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : init?.sendEncodings ? transceiver.sender?.track : null;
const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
if (kind === 'audio') {
rememberBusinessAudioSender(wrapBusinessSender(transceiver.sender), track || transceiver.sender?.track, 'pc-addTransceiver');
}
}
return transceiver;
};
}
try {
const onTrackDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
@@ -526,7 +758,7 @@
}
bridgeRemoteVideoTrack(foundVideoTrack);
bridgeRemoteAudioTrack(foundAudioTrack, 'scan');
bridgeRemoteAudioTrack(foundAudioTrack, 'scan').catch(e => console.warn(e));
}, 1000);
// ==========================================
@@ -544,39 +776,20 @@
realStream.__internal_bypass = true;
realStream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
const mixedDest = audioCtx.createMediaStreamDestination();
mixedDest.stream.__internal_bypass = true;
mixedDest.stream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
if (realStream.getAudioTracks().length > 0) {
const localMicSource = audioCtx.createMediaStreamSource(realStream);
micGainNode = audioCtx.createGain();
micGainNode.gain.value = isAiTakeover ? 0 : 1.0;
localMicSource.connect(micGainNode);
micGainNode.connect(mixedDest);
}
aiOutGainNode = audioCtx.createGain();
aiOutGainNode.gain.value = isAiTakeover ? 1.0 : 0;
aiOutGainNode.connect(mixedDest);
audioCtx = createAudioContext();
micGainNode = null;
aiOutGainNode = null;
aiLocalGainNode = audioCtx.createGain();
aiLocalGainNode.gain.value = isAiTakeover ? 1.0 : 0;
aiLocalGainNode.connect(audioCtx.destination);
if (constraints.video && realStream.getVideoTracks().length > 0) {
mixedDest.stream.addTrack(realStream.getVideoTracks()[0]);
}
if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') {
bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending');
bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending').catch(e => console.warn(e));
}
renderControlPanelSoon();
return mixedDest.stream;
return realStream;
} catch (err) {
console.error("🚨 拦截失败:", err);
return originalGetUserMedia(constraints);
@@ -598,6 +811,9 @@
if(statusEl) statusEl.textContent = "🟡 协商中...";
appendLog("🔄 正在建立 WebRTC 连接...");
pc.onconnectionstatechange = () => appendLog(`🔌 AI PC connectionState=${pc.connectionState}`);
pc.oniceconnectionstatechange = () => appendLog(`🧊 AI PC iceConnectionState=${pc.iceConnectionState}`);
initMeterAudioCtx();
const activeStream = new MediaStream([washedUploadTrack]);
const audioTransceiver = pc.addTransceiver(washedUploadTrack, {
@@ -605,7 +821,10 @@
streams: [activeStream]
});
audioSender = audioTransceiver.sender;
appendLog("🎤 已挂载洗流音频管线 (附带 PCM 强刷保障)", true);
appendLog("🎤 已挂载 AI 输入音频通道 (优先直连,失败回退洗流)", true);
if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') {
bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'sender-ready').catch(e => console.warn(e));
}
const videoTransceiver = pc.addTransceiver('video', { direction: 'sendonly' });
videoSender = videoTransceiver.sender;
@@ -615,16 +834,17 @@
pc.ontrack = (e) => {
if (e.track.kind === 'audio' && audioCtx) {
appendLog("🎧 接收到原生 AI 语音流,接入本地拓扑", true);
appendLog("🎧 接收到原生 AI 语音流,准备直连业务通话", true);
window.__ignoredTrackIds.add(e.track.id);
aiRemoteAudioTrack = e.track;
const aiStream = new MediaStream([e.track]);
aiStream.__internal_bypass = true;
const aiStreamSource = audioCtx.createMediaStreamSource(aiStream);
aiStreamSource.connect(aiOutGainNode);
aiStreamSource.connect(aiLocalGainNode);
if (aiLocalGainNode) aiStreamSource.connect(aiLocalGainNode);
window.__aiStreamSource = aiStreamSource;
attachAiTrackToBusiness();
}
};
@@ -715,6 +935,12 @@
}
function disconnectWebRTC() {
if (directInputHealthTimer) {
clearTimeout(directInputHealthTimer);
directInputHealthTimer = null;
}
aiInputMode = 'washed';
directInputTrackId = null;
if (dc) {
try { dc.close(); } catch(e){}
dc = null;
@@ -758,7 +984,7 @@
<div style="margin-bottom: 10px;">
<label style="display: block; font-size: 12px; color: #9ca3af; margin-bottom: 5px;">WebSocket 信令地址 (/ws/voice):</label>
<input type="text" id="ws-url-input" value="ws://182.92.86.220:8000/ws/voice" style="width: 100%; padding: 8px; border-radius: 4px; border: 1px solid #4b5563; background: #1f2937; color: white; box-sizing: border-box; font-size: 13px;">
<input type="text" id="ws-url-input" value="wss://182.92.86.220:8000/ws/voice" style="width: 100%; padding: 8px; border-radius: 4px; border: 1px solid #4b5563; background: #1f2937; color: white; box-sizing: border-box; font-size: 13px;">
</div>
<div style="margin-bottom: 15px;">
@@ -891,7 +1117,7 @@
fileInput.click();
});
btnToggleTakeover.addEventListener('click', () => {
btnToggleTakeover.addEventListener('click', async () => {
if (!audioCtx) {
alert("音频路由未初始化!请先在网页内加入会议。");
return;
@@ -907,8 +1133,6 @@
const now = audioCtx.currentTime;
if (isAiTakeover) {
if(micGainNode) micGainNode.gain.setValueAtTime(0, now);
if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(1.0, now);
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(1.0, now);
btnToggleTakeover.textContent = "🤖 AI 托管中 (点击切回人工)";
@@ -921,10 +1145,13 @@
}
connectWebRTC(urlInput.value, asstId);
}
if (!businessAudioSender) {
appendLog("⚠️ 尚未捕获业务通话 audio sender等待网页建立上行轨道");
}
await attachAiTrackToBusiness();
} else {
if(micGainNode) micGainNode.gain.setValueAtTime(1.0, now);
if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(0, now);
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(0, now);
await restoreBusinessAudioTrack();
btnToggleTakeover.textContent = "👨‍💼 人工通话中 (点击 AI 托管)";
btnToggleTakeover.style.background = "#10b981";