wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Audio Sender Sine Tester script for WebRTC audio management

Browse Source 
- Introduce a new user script `audio-sender-sine-tester.js` that lists current WebRTC audio senders and allows temporary sine wave transmission through selected senders.
- Implement functionality for managing audio sender candidates, tracking audio statistics, and providing real-time logging of audio activities.
- Enhance audio testing capabilities with sine wave generation and track replacement features, improving overall audio management in the WebRTC environment.
- This addition supports better monitoring and testing of audio streams in applications utilizing WebRTC.
This commit is contained in:
Xin Wang
2026-07-09 13:33:02 +08:00
parent 68931c9266
commit 03478fa557
4 changed files with 640 additions and 3309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

371
scripts/audio-sender-sine-tester.js Normal file
View File

@@ -0,0 +1,371 @@
// ==UserScript==
// @name 12345 Audio Sender Sine Tester
// @namespace http://tampermonkey.net/
// @version 1.0
// @description List current WebRTC audio senders and temporarily send a sine wave through a selected sender.
// @author You
// @match *://yhy-yw.22sj.cn:*/*
// @grant none
// @run-at document-start
// ==/UserScript==
(function () {
'use strict';
const originalAddTrack = RTCPeerConnection.prototype.addTrack;
const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
const senderCandidates = new Map();
const outboundStatsMemory = new Map();
let activeTest = null;
let lastRenderAt = 0;
function nowText() {
return new Date().toLocaleTimeString([], { hour12: false });
}
function shortId(id) {
if (!id) return 'null';
return id.length <= 18 ? id : `${id.slice(0, 8)}...${id.slice(-6)}`;
}
function log(message) {
console.log(`[Audio Sine Tester] ${message}`);
const box = document.getElementById('audio-sine-tester-log');
if (!box) return;
const row = document.createElement('div');
row.textContent = `[${nowText()}] ${message}`;
box.appendChild(row);
while (box.children.length > 80) box.removeChild(box.firstChild);
box.scrollTop = box.scrollHeight;
}
function ensureSenderCandidate(sender, track, source) {
if (!sender) return null;
const senderTrack = track || sender.track;
if (senderTrack && senderTrack.kind !== 'audio') return null;
const id = sender.__audioSineTesterId || `sender-${senderCandidates.size + 1}`;
sender.__audioSineTesterId = id;
let candidate = senderCandidates.get(id);
if (!candidate) {
candidate = {
id,
sender,
track: senderTrack || null,
originalTrack: senderTrack || null,
sources: new Set(),
bytesSent: 0,
packetsSent: 0,
bytesDelta: 0,
packetsDelta: 0,
score: 0,
};
senderCandidates.set(id, candidate);
log(`found audio sender ${id} (${source}) track=${senderTrack?.id || 'null'}`);
}
candidate.sender = sender;
candidate.track = sender.track || senderTrack || candidate.track;
if (sender.track && sender.track.kind === 'audio' && !sender.track.__audioSineTesterTrack) {
candidate.originalTrack = sender.track;
}
candidate.sources.add(source);
wrapSender(sender);
return candidate;
}
function wrapSender(sender) {
if (!sender || sender.__audioSineTesterWrapped) return;
const originalReplaceTrack = sender.replaceTrack?.bind(sender);
if (!originalReplaceTrack) return;
sender.replaceTrack = async function (track) {
const result = await originalReplaceTrack(track);
const candidate = ensureSenderCandidate(sender, track, 'replaceTrack');
if (candidate) {
candidate.track = track || sender.track || candidate.track;
if (track && !track.__audioSineTesterTrack) candidate.originalTrack = track;
log(`${candidate.id} replaceTrack -> ${track?.id || 'null'}`);
}
return result;
};
sender.__audioSineTesterWrapped = true;
}
async function refreshSenderStats() {
for (const candidate of senderCandidates.values()) {
const sender = candidate.sender;
candidate.track = sender.track || candidate.track;
if (!sender.getStats) continue;
try {
const report = await sender.getStats();
report.forEach((stat) => {
const kind = stat.kind || stat.mediaType;
if (stat.type !== 'outbound-rtp' || kind !== 'audio') return;
const key = `${candidate.id}:${stat.id}`;
const prev = outboundStatsMemory.get(key);
const bytes = stat.bytesSent || 0;
const packets = stat.packetsSent || 0;
candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
candidate.bytesSent = bytes;
candidate.packetsSent = packets;
outboundStatsMemory.set(key, { bytes, packets });
candidate.score = scoreCandidate(candidate);
});
} catch (error) {
candidate.sources.add(`stats-error:${error.message || error}`);
}
}
}
function scoreCandidate(candidate) {
let score = 0;
if (candidate.track?.kind === 'audio') score += 1000;
if (candidate.track?.readyState === 'live') score += 800;
if (candidate.sources.has('pc-addTrack')) score += 1500;
if (candidate.bytesSent > 0) score += 600;
if (candidate.bytesDelta > 0) score += Math.min(2500, candidate.bytesDelta * 4);
if (candidate.packetsDelta > 0) score += Math.min(1000, candidate.packetsDelta * 12);
return score;
}
function makeSineTrack(frequency) {
const AudioCtx = window.AudioContext || window.webkitAudioContext;
const context = new AudioCtx({ sampleRate: 48000 });
const oscillator = context.createOscillator();
const gain = context.createGain();
const dest = context.createMediaStreamDestination();
oscillator.type = 'sine';
oscillator.frequency.value = frequency;
gain.gain.value = 0.08;
oscillator.connect(gain);
gain.connect(dest);
oscillator.start();
const track = dest.stream.getAudioTracks()[0];
track.__audioSineTesterTrack = true;
track.__audioSineTesterContext = context;
track.__audioSineTesterOscillator = oscillator;
track.__audioSineTesterGain = gain;
return track;
}
function stopTrack(track) {
if (!track) return;
try { track.__audioSineTesterOscillator?.stop(); } catch {}
try { track.stop(); } catch {}
try { track.__audioSineTesterContext?.close(); } catch {}
}
async function startSine(candidateId, frequency) {
const candidate = senderCandidates.get(candidateId);
if (!candidate?.sender) return;
await stopSine(false);
const originalTrack = candidate.sender.track || candidate.originalTrack || null;
const sineTrack = makeSineTrack(frequency);
await candidate.sender.replaceTrack(sineTrack);
activeTest = {
candidateId,
sender: candidate.sender,
originalTrack,
sineTrack,
frequency,
startedAt: Date.now(),
};
candidate.originalTrack = originalTrack;
candidate.track = sineTrack;
log(`START ${frequency}Hz on ${candidateId}; original=${originalTrack?.id || 'null'}`);
renderPanel();
}
async function stopSine(shouldLog = true) {
if (!activeTest) return;
const test = activeTest;
activeTest = null;
try {
await test.sender.replaceTrack(test.originalTrack || null);
if (shouldLog) log(`STOP ${test.frequency}Hz on ${test.candidateId}; restored=${test.originalTrack?.id || 'null'}`);
} catch (error) {
log(`restore failed on ${test.candidateId}: ${error.message || error}`);
} finally {
stopTrack(test.sineTrack);
}
renderPanel();
}
function dump() {
return {
activeTest: activeTest ? {
candidateId: activeTest.candidateId,
frequency: activeTest.frequency,
sineTrackId: activeTest.sineTrack.id,
originalTrackId: activeTest.originalTrack?.id || null,
seconds: Math.round((Date.now() - activeTest.startedAt) / 1000),
} : null,
senders: Array.from(senderCandidates.values())
.sort((a, b) => b.score - a.score)
.map((candidate) => ({
id: candidate.id,
active: activeTest?.candidateId === candidate.id,
trackId: candidate.sender.track?.id || candidate.track?.id || null,
trackLabel: candidate.sender.track?.label || candidate.track?.label || null,
originalTrackId: candidate.originalTrack?.id || null,
trackEnabled: candidate.sender.track?.enabled ?? null,
trackMuted: candidate.sender.track?.muted ?? null,
trackReadyState: candidate.sender.track?.readyState || null,
bytesSent: candidate.bytesSent,
packetsSent: candidate.packetsSent,
bytesDelta: candidate.bytesDelta,
packetsDelta: candidate.packetsDelta,
score: Math.round(candidate.score),
sources: Array.from(candidate.sources),
})),
};
}
function renderPanel() {
if (!document.body) return;
let panel = document.getElementById('audio-sine-tester-panel');
if (!panel) {
panel = document.createElement('div');
panel.id = 'audio-sine-tester-panel';
panel.style.cssText = [
'position:fixed',
'top:20px',
'right:20px',
'width:500px',
'max-height:76vh',
'z-index:999999',
'box-sizing:border-box',
'padding:12px',
'border-radius:8px',
'background:rgba(3,7,18,.96)',
'color:#e5e7eb',
'font:12px ui-monospace,Menlo,monospace',
'box-shadow:0 12px 32px rgba(0,0,0,.35)',
'overflow:auto',
].join(';');
panel.innerHTML = `
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:8px;">
<strong style="color:#fcd34d;font:13px system-ui,sans-serif;">Audio Sender Sine Tester</strong>
<button id="audio-sine-copy" style="border:0;border-radius:5px;padding:5px 8px;background:#2563eb;color:white;cursor:pointer;">copy dump</button>
</div>
<div id="audio-sine-summary" style="margin-bottom:8px;color:#d1d5db;"></div>
<div id="audio-sine-list"></div>
<div style="display:flex;gap:8px;margin-top:8px;">
<button id="audio-sine-stop" style="flex:1;border:0;border-radius:5px;padding:8px;background:#dc2626;color:white;cursor:pointer;">stop and restore</button>
</div>
<div id="audio-sine-tester-log" style="margin-top:8px;height:90px;overflow:auto;border-top:1px solid #374151;padding-top:6px;color:#a7f3d0;"></div>
`;
document.body.appendChild(panel);
document.getElementById('audio-sine-copy').addEventListener('click', async () => {
const text = JSON.stringify(window.__audioSineTesterDump(), null, 2);
await navigator.clipboard?.writeText(text).catch(() => {});
console.log('[Audio Sine Tester dump]', text);
log('dump copied to clipboard and console');
});
document.getElementById('audio-sine-stop').addEventListener('click', () => stopSine());
}
const data = dump();
document.getElementById('audio-sine-summary').textContent =
`senders=${data.senders.length}, active=${data.activeTest ? `${data.activeTest.candidateId} ${data.activeTest.frequency}Hz ${data.activeTest.seconds}s` : 'none'}, time=${nowText()}`;
document.getElementById('audio-sine-list').innerHTML = data.senders.map((row) => {
const pct = Math.min(100, Math.max(row.bytesDelta / 20, row.packetsDelta * 2));
const active = row.active;
return `
<div style="border:1px solid ${active ? '#f59e0b' : '#374151'};border-radius:6px;padding:8px;margin-bottom:6px;">
<div style="display:flex;justify-content:space-between;gap:8px;color:${active ? '#fcd34d' : '#cbd5e1'};">
<span>${active ? '*' : '-'} ${row.id}</span>
<span>score=${row.score}</span>
</div>
<div style="height:6px;background:#374151;border-radius:999px;overflow:hidden;margin:6px 0;">
<div style="height:100%;width:${pct}%;background:linear-gradient(90deg,#60a5fa,#fcd34d);"></div>
</div>
<div style="color:#cbd5e1;line-height:1.45;word-break:break-all;">
track=${shortId(row.trackId)} label=${row.trackLabel || 'null'} muted=${row.trackMuted} bytesDelta=${row.bytesDelta} packetsDelta=${row.packetsDelta}
</div>
<div style="display:flex;gap:6px;margin-top:6px;">
<button data-sine="${row.id}:440" style="border:0;border-radius:5px;padding:6px 8px;background:#059669;color:white;cursor:pointer;">440Hz</button>
<button data-sine="${row.id}:880" style="border:0;border-radius:5px;padding:6px 8px;background:#7c3aed;color:white;cursor:pointer;">880Hz</button>
</div>
<div style="color:#94a3b8;margin-top:5px;word-break:break-all;">${row.sources.join(', ')}</div>
</div>
`;
}).join('');
panel.querySelectorAll('[data-sine]').forEach((button) => {
button.onclick = () => {
const [id, hz] = button.dataset.sine.split(':');
startSine(id, Number(hz));
};
});
}
RTCPeerConnection.prototype.addTrack = function (track, ...streams) {
const sender = originalAddTrack.call(this, track, ...streams);
if (track?.kind === 'audio') ensureSenderCandidate(sender, track, 'pc-addTrack');
return sender;
};
if (originalAddTransceiver) {
RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) {
const transceiver = originalAddTransceiver.call(this, trackOrKind, init);
const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
if (kind === 'audio') {
const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track;
ensureSenderCandidate(transceiver.sender, track, 'pc-addTransceiver');
}
return transceiver;
};
}
RTCPeerConnection.prototype.setRemoteDescription = function () {
const result = originalSetRemoteDescription.apply(this, arguments);
Promise.resolve(result).then(() => {
try {
for (const sender of this.getSenders?.() || []) {
ensureSenderCandidate(sender, sender.track, 'sender-after-srd');
}
} catch (error) {
console.warn('[Audio Sine Tester] failed to scan senders', error);
}
});
return result;
};
window.__audioSineTesterDump = dump;
window.__audioSineTesterStop = stopSine;
setInterval(refreshSenderStats, 250);
setInterval(() => {
if (Date.now() - lastRenderAt < 500) return;
lastRenderAt = Date.now();
renderPanel();
}, 250);
window.addEventListener('beforeunload', () => {
if (activeTest) stopSine(false);
});
if (document.readyState === 'loading') {
document.addEventListener('DOMContentLoaded', renderPanel, { once: true });
} else {
renderPanel();
}
})();

1725
scripts/copilot-native-direct-output.js
View File

File diff suppressed because it is too large Load Diff

1542
scripts/copilot-native.js
View File

File diff suppressed because it is too large Load Diff

311
scripts/copilot.js
View File

@@ -1,5 +1,5 @@
// ==UserScript==
// @name 12345 AI 协同中台 (WebRTC 洗流桥接版)
// @name 12345 AI 协同中台
// @namespace http://tampermonkey.net/
// @version 7.5
// @description 引入 PCM ScriptProcessor 强制软解防丢包Bug；增加真实洗流耳返监听功能，彻底解决远端触发失败悬案
@@ -23,6 +23,8 @@
}
const origAddEventListener = RTCPeerConnection.prototype.addEventListener;
const origSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
const origAddTrack = RTCPeerConnection.prototype.addTrack;
const origAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
const origAddStream = RTCPeerConnection.prototype.addStream;
const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
@@ -43,6 +45,13 @@
let audioSender = null;
let videoSender = null;
let analyserNode = null;
let businessAudioSender = null;
let businessOriginalAudioTrack = null;
let aiRemoteAudioTrack = null;
let aiTrackAttachedToBusiness = false;
let aiInputMode = 'washed';
let directInputHealthTimer = null;
let directInputTrackId = null;
let isAiTakeover = false;
let isMonitoring = false;
@@ -59,6 +68,15 @@
const remoteReceiversByTrackId = new Map();
const receiverStatsMemory = new Map();
const capturedVideoElementStreams = new WeakMap();
function createAudioContext() {
const AudioContextClass = window.AudioContext || window.webkitAudioContext;
try {
return new AudioContextClass({ sampleRate: 48000 });
} catch (e) {
console.warn("48k AudioContext 创建失败，回退浏览器默认采样率:", e);
return new AudioContextClass();
}
}
function isInternalPeerConnection(target) {
return !!(target && target.__aiCopilotInternal);
@@ -98,10 +116,193 @@
console.log(`[AI Copilot WebRTC] 绑定远端 receiver(${reason})`, track.kind, track.id);
}
function rememberBusinessAudioSender(sender, track, reason = 'unknown') {
if (!sender) return;
const senderTrack = track || sender.track;
if (senderTrack && senderTrack.kind !== 'audio') return;
businessAudioSender = sender;
if (senderTrack && senderTrack !== aiRemoteAudioTrack) {
businessOriginalAudioTrack = senderTrack;
}
console.log(`[AI Copilot WebRTC] 绑定业务通话 audio sender(${reason})`, senderTrack?.id || 'no-track-yet');
if (isAiTakeover && aiRemoteAudioTrack) attachAiTrackToBusiness();
}
function wrapBusinessSender(sender) {
if (!sender || sender.__aiCopilotWrappedSender) return sender;
const originalReplaceTrack = sender.replaceTrack?.bind(sender);
if (!originalReplaceTrack) return sender;
sender.replaceTrack = async function(track) {
const ret = await originalReplaceTrack(track);
if (track && track.kind === 'audio' && track !== aiRemoteAudioTrack && !isAiTakeover) {
rememberBusinessAudioSender(sender, track, 'sender-replaceTrack');
}
return ret;
};
sender.__aiCopilotWrappedSender = true;
return sender;
}
async function attachAiTrackToBusiness() {
if (!isAiTakeover || !businessAudioSender || !aiRemoteAudioTrack || aiRemoteAudioTrack.readyState !== 'live') return false;
try {
await businessAudioSender.replaceTrack(aiRemoteAudioTrack);
aiTrackAttachedToBusiness = true;
appendLog("🔁 已将 AI 原生音轨直接替换到业务通话上行", true);
return true;
} catch (e) {
aiTrackAttachedToBusiness = false;
appendLog(`⚠️ AI 原生音轨替换业务 sender 失败: ${e.message || e}`);
return false;
}
}
async function restoreBusinessAudioTrack() {
if (!businessAudioSender || !aiTrackAttachedToBusiness) return;
try {
await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null);
appendLog("👨‍💼 已恢复业务通话原始音轨", true);
} catch (e) {
appendLog(`⚠️ 恢复业务音轨失败: ${e.message || e}`);
} finally {
aiTrackAttachedToBusiness = false;
}
}
async function outboundAudioBytes(sender) {
if (!sender?.getStats) return null;
try {
let total = 0;
const report = await sender.getStats();
report.forEach(stat => {
const statKind = stat.kind || stat.mediaType;
if (stat.type === 'outbound-rtp' && statKind === 'audio' && typeof stat.bytesSent === 'number') {
total += stat.bytesSent;
}
});
return total;
} catch (e) {
return null;
}
}
function aiPeerReadyForMedia() {
if (!pc || !pc.remoteDescription) return false;
return (
pc.connectionState === 'connected'
|| pc.iceConnectionState === 'connected'
|| pc.iceConnectionState === 'completed'
);
}
function aiPeerStateLabel() {
if (!pc) return 'pc=null';
return `conn=${pc.connectionState},ice=${pc.iceConnectionState},signaling=${pc.signalingState},remote=${pc.remoteDescription?.type || 'none'}`;
}
function waitForAiPeerMediaReady(timeoutMs = 10000) {
if (aiPeerReadyForMedia()) return Promise.resolve(true);
return new Promise((resolve) => {
const startedAt = Date.now();
const timer = setInterval(() => {
if (aiPeerReadyForMedia()) {
clearInterval(timer);
resolve(true);
} else if (Date.now() - startedAt >= timeoutMs) {
clearInterval(timer);
resolve(false);
}
}, 250);
});
}
function disconnectWashedInputGraph() {
if (currentMeterSourceNode) {
try { currentMeterSourceNode.disconnect(); } catch(e) {}
currentMeterSourceNode = null;
}
if (currentVadBoostGainNode) {
try { currentVadBoostGainNode.disconnect(); } catch(e) {}
currentVadBoostGainNode = null;
}
if (pcmForcerNode) {
try { pcmForcerNode.disconnect(); } catch(e) {}
}
}
async function fallbackToWashedAiInput(track, reason = 'fallback') {
if (!track || track.readyState !== 'live') return;
if (directInputHealthTimer) {
clearTimeout(directInputHealthTimer);
directInputHealthTimer = null;
}
directInputTrackId = null;
if (audioSender && washedUploadTrack && audioSender.track !== washedUploadTrack) {
try {
await audioSender.replaceTrack(washedUploadTrack);
} catch (e) {
appendLog(`⚠️ AI 输入切回洗流轨失败: ${e.message || e}`);
}
}
aiInputMode = 'washed';
appendLog(`🧯 AI 输入切回洗流链路(${reason})`, true);
ensureVuMeterForTrack(track, reason);
}
async function tryDirectAiInput(track, reason = 'direct') {
if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return false;
if (!audioSender) {
pendingBridgeAudioTrack = track;
return false;
}
if (aiInputMode === 'direct' && directInputTrackId === track.id && audioSender.track === track) return true;
try {
if (directInputHealthTimer) clearTimeout(directInputHealthTimer);
await audioSender.replaceTrack(track);
aiInputMode = 'direct';
directInputTrackId = track.id;
disconnectWashedInputGraph();
appendLog(`🚀 AI 输入直连远端原生音轨(${reason}); track=${track.id}, ${aiPeerStateLabel()}`, true);
directInputHealthTimer = setTimeout(async () => {
if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
appendLog(`🩺 等待 AI WebRTC 媒体连接后检查直连输入: ${aiPeerStateLabel()}`);
const ready = await waitForAiPeerMediaReady(10000);
if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
if (!ready) {
appendLog(`⚠️ AI WebRTC 媒体连接超时，直连输入无法验证: ${aiPeerStateLabel()}`);
await fallbackToWashedAiInput(track, 'direct-ai-pc-not-ready');
return;
}
const beforeHealthBytes = await outboundAudioBytes(audioSender);
await new Promise(resolve => setTimeout(resolve, 2500));
if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return;
const afterBytes = await outboundAudioBytes(audioSender);
appendLog(`🩺 直连输入发包检查: before=${beforeHealthBytes}, after=${afterBytes}, ${aiPeerStateLabel()}`);
if (
afterBytes !== null
&& (
(beforeHealthBytes !== null && afterBytes <= beforeHealthBytes)
|| (beforeHealthBytes === null && afterBytes === 0)
)
) {
await fallbackToWashedAiInput(track, 'direct-no-outbound-audio');
}
}, 500);
return true;
} catch (e) {
appendLog(`⚠️ AI 输入直连失败，改用洗流: ${e.message || e}`);
await fallbackToWashedAiInput(track, 'direct-replace-failed');
return false;
}
}
function initMeterAudioCtx() {
if (meterAudioCtx) return;
try {
meterAudioCtx = new (window.AudioContext || window.webkitAudioContext)();
meterAudioCtx = createAudioContext();
meterKeepAliveGainNode = meterAudioCtx.createGain();
meterKeepAliveGainNode.gain.value = 0;
@@ -134,14 +335,23 @@
}
}
function bridgeRemoteAudioTrack(track, reason = 'scan') {
async function bridgeRemoteAudioTrack(track, reason = 'scan') {
if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return;
if (track === currentStolenAudioTrack && currentMeterSourceNode) return;
if (
track === currentStolenAudioTrack
&& (
(aiInputMode === 'direct' && audioSender?.track === track)
|| (aiInputMode === 'washed' && currentMeterSourceNode)
)
) return;
pendingBridgeAudioTrack = track;
appendLog(`🎙️ 锁定远端发声轨(${reason})，接入混流总线`, true);
appendLog(`🎙️ 锁定远端发声轨(${reason})，尝试直连 AI 输入`, true);
currentStolenAudioTrack = track;
ensureVuMeterForTrack(track, reason);
const directOk = await tryDirectAiInput(track, reason);
if (!directOk && audioSender) {
await fallbackToWashedAiInput(track, `${reason}-fallback`);
}
}
function capturePlayableElementVideoStream(el) {
@@ -196,7 +406,7 @@
// 2. AGC 放大增益 (击穿 VAD 阈值)
const vadBoostGain = meterAudioCtx.createGain();
vadBoostGain.gain.value = 8.0; // 放大 8 倍
vadBoostGain.gain.value = 2.0; // fallback 洗流仅轻度放大，避免底噪/回声被过度抬高
currentVadBoostGainNode = vadBoostGain;
currentMeterSourceNode.connect(vadBoostGain);
@@ -377,6 +587,28 @@
return origAddEventListener.call(this, type, listener, options);
};
RTCPeerConnection.prototype.addTrack = function(track, ...streams) {
const sender = origAddTrack.call(this, track, ...streams);
if (!isInternalPeerConnection(this) && track?.kind === 'audio') {
rememberBusinessAudioSender(wrapBusinessSender(sender), track, 'pc-addTrack');
}
return sender;
};
if (origAddTransceiver) {
RTCPeerConnection.prototype.addTransceiver = function(trackOrKind, init) {
const transceiver = origAddTransceiver.call(this, trackOrKind, init);
if (!isInternalPeerConnection(this)) {
const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : init?.sendEncodings ? transceiver.sender?.track : null;
const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
if (kind === 'audio') {
rememberBusinessAudioSender(wrapBusinessSender(transceiver.sender), track || transceiver.sender?.track, 'pc-addTransceiver');
}
}
return transceiver;
};
}
try {
const onTrackDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
@@ -526,7 +758,7 @@
}
bridgeRemoteVideoTrack(foundVideoTrack);
bridgeRemoteAudioTrack(foundAudioTrack, 'scan');
bridgeRemoteAudioTrack(foundAudioTrack, 'scan').catch(e => console.warn(e));
}, 1000);
// ==========================================
@@ -544,39 +776,20 @@
realStream.__internal_bypass = true;
realStream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
audioCtx = new (window.AudioContext || window.webkitAudioContext)();
const mixedDest = audioCtx.createMediaStreamDestination();
mixedDest.stream.__internal_bypass = true;
mixedDest.stream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id));
if (realStream.getAudioTracks().length > 0) {
const localMicSource = audioCtx.createMediaStreamSource(realStream);
micGainNode = audioCtx.createGain();
micGainNode.gain.value = isAiTakeover ? 0 : 1.0;
localMicSource.connect(micGainNode);
micGainNode.connect(mixedDest);
}
aiOutGainNode = audioCtx.createGain();
aiOutGainNode.gain.value = isAiTakeover ? 1.0 : 0;
aiOutGainNode.connect(mixedDest);
audioCtx = createAudioContext();
micGainNode = null;
aiOutGainNode = null;
aiLocalGainNode = audioCtx.createGain();
aiLocalGainNode.gain.value = isAiTakeover ? 1.0 : 0;
aiLocalGainNode.connect(audioCtx.destination);
if (constraints.video && realStream.getVideoTracks().length > 0) {
mixedDest.stream.addTrack(realStream.getVideoTracks()[0]);
}
if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') {
bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending');
bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending').catch(e => console.warn(e));
}
renderControlPanelSoon();
return mixedDest.stream;
return realStream;
} catch (err) {
console.error("🚨 拦截失败:", err);
return originalGetUserMedia(constraints);
@@ -598,6 +811,9 @@
if(statusEl) statusEl.textContent = "🟡 协商中...";
appendLog("🔄 正在建立 WebRTC 连接...");
pc.onconnectionstatechange = () => appendLog(`🔌 AI PC connectionState=${pc.connectionState}`);
pc.oniceconnectionstatechange = () => appendLog(`🧊 AI PC iceConnectionState=${pc.iceConnectionState}`);
initMeterAudioCtx();
const activeStream = new MediaStream([washedUploadTrack]);
const audioTransceiver = pc.addTransceiver(washedUploadTrack, {
@@ -605,7 +821,10 @@
streams: [activeStream]
});
audioSender = audioTransceiver.sender;
appendLog("🎤 已挂载洗流音频管线 (附带 PCM 强刷保障)", true);
appendLog("🎤 已挂载 AI 输入音频通道 (优先直连，失败回退洗流)", true);
if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') {
bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'sender-ready').catch(e => console.warn(e));
}
const videoTransceiver = pc.addTransceiver('video', { direction: 'sendonly' });
videoSender = videoTransceiver.sender;
@@ -615,16 +834,17 @@
pc.ontrack = (e) => {
if (e.track.kind === 'audio' && audioCtx) {
appendLog("🎧 接收到原生 AI 语音流，接入本地拓扑", true);
appendLog("🎧 接收到原生 AI 语音流，准备直连业务通话", true);
window.__ignoredTrackIds.add(e.track.id);
aiRemoteAudioTrack = e.track;
const aiStream = new MediaStream([e.track]);
aiStream.__internal_bypass = true;
const aiStreamSource = audioCtx.createMediaStreamSource(aiStream);
aiStreamSource.connect(aiOutGainNode);
aiStreamSource.connect(aiLocalGainNode);
if (aiLocalGainNode) aiStreamSource.connect(aiLocalGainNode);
window.__aiStreamSource = aiStreamSource;
attachAiTrackToBusiness();
}
};
@@ -715,6 +935,12 @@
}
function disconnectWebRTC() {
if (directInputHealthTimer) {
clearTimeout(directInputHealthTimer);
directInputHealthTimer = null;
}
aiInputMode = 'washed';
directInputTrackId = null;
if (dc) {
try { dc.close(); } catch(e){}
dc = null;
@@ -758,7 +984,7 @@
<div style="margin-bottom: 10px;">
<label style="display: block; font-size: 12px; color: #9ca3af; margin-bottom: 5px;">WebSocket 信令地址 (/ws/voice):</label>
<input type="text" id="ws-url-input" value="ws://182.92.86.220:8000/ws/voice" style="width: 100%; padding: 8px; border-radius: 4px; border: 1px solid #4b5563; background: #1f2937; color: white; box-sizing: border-box; font-size: 13px;">
<input type="text" id="ws-url-input" value="wss://182.92.86.220:8000/ws/voice" style="width: 100%; padding: 8px; border-radius: 4px; border: 1px solid #4b5563; background: #1f2937; color: white; box-sizing: border-box; font-size: 13px;">
</div>
<div style="margin-bottom: 15px;">
@@ -891,7 +1117,7 @@
fileInput.click();
});
btnToggleTakeover.addEventListener('click', () => {
btnToggleTakeover.addEventListener('click', async () => {
if (!audioCtx) {
alert("音频路由未初始化！请先在网页内加入会议。");
return;
@@ -907,8 +1133,6 @@
const now = audioCtx.currentTime;
if (isAiTakeover) {
if(micGainNode) micGainNode.gain.setValueAtTime(0, now);
if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(1.0, now);
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(1.0, now);
btnToggleTakeover.textContent = "🤖 AI 托管中 (点击切回人工)";
@@ -921,10 +1145,13 @@
}
connectWebRTC(urlInput.value, asstId);
}
if (!businessAudioSender) {
appendLog("⚠️ 尚未捕获业务通话 audio sender，等待网页建立上行轨道");
}
await attachAiTrackToBusiness();
} else {
if(micGainNode) micGainNode.gain.setValueAtTime(1.0, now);
if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(0, now);
if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(0, now);
await restoreBusinessAudioTrack();
btnToggleTakeover.textContent = "👨‍💼 人工通话中 (点击 AI 托管)";
btnToggleTakeover.style.background = "#10b981";