wx44wx/ai-video-fullstack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor AI Element2 Gated Copilot script for improved audio handling

Browse Source 
- Update the description to clarify that participant audio is fed into AI after both sides are connected.
- Introduce new functions to manage audio track patterns and enhance the logic for determining participant and operator presence.
- Improve gate status messaging to reflect the online status of participants and operators.
- Add volume handling functions for better audio visualization and feedback.
- Refactor existing functions to streamline audio processing and improve overall code clarity.
This commit is contained in:
Xin Wang
2026-07-09 08:54:15 +08:00
parent 4601aeab75
commit fe443932f0
1 changed files with 144 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
scripts/copilot-element2-two-party.js
View File

@@ -2,7 +2,7 @@
// @name 12345 AI Element2 Gated Copilot
// @namespace http://tampermonkey.net/
// @version 1.0
// @description Two-party gated bridge: only feed element-2:audio into AI after both sides are connected.
// @description Two-party gated bridge: only feed participant audio into AI after both sides are connected.
// @author You
// @match *://yhy-yw.22sj.cn:*/*
// @grant none
@@ -15,6 +15,7 @@
const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice';
const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio';
const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video';
const PARTICIPANT_AUDIO_TRACK_PATTERN = /-audio-3-a$/;
const originalAddEventListener = RTCPeerConnection.prototype.addEventListener;
const originalAddTrack = RTCPeerConnection.prototype.addTrack;
@@ -45,6 +46,11 @@
let aiVideoSender = null;
let isAiTakeover = false;
let aiAttached = false;
let aiOutputAudioContext = null;
let aiOutputAnalyser = null;
let aiOutputAnalyserData = null;
let aiOutputAnalyserSource = null;
let aiOutputAnalyserTrack = null;
let inputVu = 0;
let outputVu = 0;
let inputHealthTimer = null;
@@ -331,40 +337,51 @@
function updateSelectedAudioLabel() {
const label = document.getElementById('element2-copilot-selected-audio');
if (!label) return;
const item = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id);
label.textContent = item
? `${audioSourceLabel(item)} ${selectedRemoteAudio.id.slice(0, 8)} score=${Math.round(item.audioScore || 0)}`
: `等待 ${STATIC_AUDIO_ELEMENT_REASON}`;
label.textContent = '截取给 AI 的音频音量';
}
function pageHasBothParties() {
function pageHasParticipant() {
const text = document.body?.innerText || '';
return text.includes('当事人') && text.includes('坐席端');
return text.includes('当事人');
}
function pageHasOperator() {
const text = document.body?.innerText || '';
return text.includes('坐席端');
}
function isParticipantAudioItem(candidate) {
return Boolean(
candidate
&& isLive(candidate.track)
&& !candidate.track.muted
&& hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON)
&& PARTICIPANT_AUDIO_TRACK_PATTERN.test(candidate.track.id)
);
}
function staticAudioItem() {
return Array.from(remoteAudio.values()).find((candidate) => (
isLive(candidate.track)
&& !candidate.track.muted
&& hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON)
)) || null;
return Array.from(remoteAudio.values()).find(isParticipantAudioItem) || null;
}
function gateStatus() {
const item = staticAudioItem();
const participantVisible = pageHasParticipant();
const operatorVisible = pageHasOperator();
return {
bothParties: pageHasBothParties(),
audioReady: Boolean(item),
bothParties: participantVisible && operatorVisible,
participantOnline: participantVisible && Boolean(item),
operatorOnline: operatorVisible,
senderReady: Boolean(selectedBusinessSender),
track: item?.track || null,
};
}
function gateMessage(status = gateStatus()) {
if (!status.bothParties) return '等待当事人和坐席端同时在线';
if (!status.audioReady) return `等待 ${STATIC_AUDIO_ELEMENT_REASON} 有声轨`;
if (!status.senderReady) return '等待业务上行 sender';
return `就绪：${STATIC_AUDIO_ELEMENT_REASON} -> AI`;
if (!status.operatorOnline) return '等待坐席在线';
if (!status.participantOnline) return '等待当事人在线';
if (!status.senderReady) return '等待通话上行就绪';
return '就绪：可开启 AI 托管';
}
function updateGateLabel() {
@@ -372,7 +389,7 @@
if (!label) return;
const status = gateStatus();
label.textContent = gateMessage(status);
label.style.color = status.bothParties && status.audioReady && status.senderReady ? '#86efac' : '#fcd34d';
label.style.color = status.participantOnline && status.operatorOnline && status.senderReady ? '#86efac' : '#fcd34d';
}
function scoreAudioCandidate(item) {
@@ -448,7 +465,7 @@
function selectRemoteAudio(reason) {
const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id);
if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted && hasElementReason(current, STATIC_AUDIO_ELEMENT_REASON)) {
if (isParticipantAudioItem(current)) {
updateGateLabel();
return selectedRemoteAudio;
}
@@ -549,17 +566,113 @@
function setVu(id, level) {
const el = document.getElementById(id);
if (el) el.style.width = `${Math.min(100, level * 2500)}%`;
if (!el) return;
el.style.width = `${Math.min(100, level * 2500)}%`;
el.style.opacity = level > 0 ? '1' : '0.35';
}
function volumePercent(level) {
if (!Number.isFinite(level) || level < 0.0015) return 0;
return Math.round(Math.min(100, level * 2500));
}
function volumeDb(level) {
if (!Number.isFinite(level) || level < 0.0015) return '-inf';
return `${(20 * Math.log10(level)).toFixed(1)}dB`;
}
function volumeText(level) {
return `vol=${volumePercent(level)}% ${volumeDb(level)}`;
}
function setVolumeLabel(id, level) {
const el = document.getElementById(id);
if (!el) return;
el.textContent = volumeText(level);
}
function renderVu() {
refreshAiOutputVuFromAnalyser();
inputVu *= 0.88;
outputVu *= 0.88;
setVu('element2-copilot-input-vu', inputVu);
setVu('element2-copilot-output-vu', outputVu);
setVolumeLabel('element2-copilot-input-volume', inputVu);
setVolumeLabel('element2-copilot-output-volume', outputVu);
requestAnimationFrame(renderVu);
}
function speechVuLevel(level) {
const noiseFloor = 0.0015;
if (!Number.isFinite(level) || level < noiseFloor) return 0;
return Math.min(0.08, (level - noiseFloor) * 3.2);
}
function analyserVuLevel(analyser, data) {
if (!analyser || !data) return 0;
analyser.getByteTimeDomainData(data);
let sum = 0;
for (const value of data) {
const centered = (value - 128) / 128;
sum += centered * centered;
}
return speechVuLevel(Math.sqrt(sum / data.length));
}
function refreshAiOutputVuFromAnalyser() {
outputVu = Math.max(outputVu, analyserVuLevel(aiOutputAnalyser, aiOutputAnalyserData));
}
function createAudioContext() {
const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
return AudioContextCtor ? new AudioContextCtor() : null;
}
function detachAiOutputMeter() {
try {
aiOutputAnalyserSource?.disconnect();
} catch {}
aiOutputAnalyser = null;
aiOutputAnalyserData = null;
aiOutputAnalyserSource = null;
aiOutputAnalyserTrack = null;
outputVu = 0;
}
async function attachAiOutputMeter(track) {
if (!isLive(track)) {
detachAiOutputMeter();
return false;
}
if (aiOutputAnalyserTrack === track && aiOutputAnalyser) return true;
detachAiOutputMeter();
if (!aiOutputAudioContext) {
aiOutputAudioContext = createAudioContext();
if (!aiOutputAudioContext) {
appendChatLine('音频', '当前浏览器不支持 AudioContext，无法显示 AI 输出音量', true);
return false;
}
}
if (aiOutputAudioContext.state === 'suspended') {
await aiOutputAudioContext.resume().catch(() => {});
}
try {
const stream = new MediaStream([track]);
aiOutputAnalyserSource = aiOutputAudioContext.createMediaStreamSource(stream);
aiOutputAnalyser = aiOutputAudioContext.createAnalyser();
aiOutputAnalyser.fftSize = 512;
aiOutputAnalyser.smoothingTimeConstant = 0.55;
aiOutputAnalyserData = new Uint8Array(aiOutputAnalyser.fftSize);
aiOutputAnalyserSource.connect(aiOutputAnalyser);
aiOutputAnalyserTrack = track;
return true;
} catch (error) {
log(`failed to attach AI output meter: ${error.message || error}`);
detachAiOutputMeter();
return false;
}
}
async function refreshStats() {
await refreshInputStats();
await refreshOutputStats();
@@ -601,7 +714,7 @@
duration: stat.totalSamplesDuration,
});
if (item.track === selectedRemoteAudio) {
inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500), Math.min(0.18, audioLevel));
inputVu = Math.max(inputVu, speechVuLevel(audioLevel));
}
});
}
@@ -625,9 +738,6 @@
item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
outboundMemory.set(key, { bytes, packets });
if (item.sender === selectedBusinessSender) {
outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500));
}
});
}
}
@@ -846,7 +956,7 @@
selectBusinessSender('connect');
const status = gateStatus();
updateGateLabel();
if (!status.bothParties || !status.audioReady || !status.senderReady) {
if (!status.operatorOnline || !status.participantOnline || !status.senderReady) {
appendChatLine('门槛', gateMessage(status), true);
setStatus('disconnected');
return false;
@@ -917,6 +1027,7 @@
aiOutputTrackIds.add(event.track.id);
remoteAudio.delete(event.track.id);
aiAudioTrack = event.track;
attachAiOutputMeter(event.track);
log(`received AI audio ${trackLabel(aiAudioTrack)}`, true);
aiAudioTrack.onunmute = () => {
log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`);
@@ -993,6 +1104,7 @@
aiAudioSender = null;
aiVideoSender = null;
aiAudioTrack = null;
detachAiOutputMeter();
if (inputRecorder?.state === 'recording') stopInputRecording();
if (videoRecorder?.state === 'recording') stopVideoRecording();
setStatus('disconnected');
@@ -1028,7 +1140,8 @@
staticVideoElement: STATIC_VIDEO_ELEMENT_REASON,
gate: {
bothParties: status.bothParties,
audioReady: status.audioReady,
participantOnline: status.participantOnline,
operatorOnline: status.operatorOnline,
senderReady: status.senderReady,
track: trackLabel(status.track),
},
@@ -1112,13 +1225,11 @@
].join(';');
panel.innerHTML = `
<div style="display:flex;justify-content:space-between;gap:12px;align-items:center;margin-bottom:12px;">
<strong style="font-size:15px;color:#f9a8d4;">AI 元素选轨</strong>
<strong style="font-size:15px;color:#f9a8d4;">AI 托管</strong>
<span id="element2-copilot-status" style="font-size:12px;color:#d1d5db;">未连接</span>
</div>
<div style="font-size:11px;color:#9ca3af;margin-bottom:10px;line-height:1.5;">
音频输入：${STATIC_AUDIO_ELEMENT_REASON}<br>
视频输入：${STATIC_VIDEO_ELEMENT_REASON}<br>
<span id="element2-copilot-gate" style="color:#fcd34d;">等待当事人和坐席端同时在线</span>
<span id="element2-copilot-gate" style="color:#fcd34d;">等待当事人在线</span>
</div>
<label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">信令地址</label>
<input id="element2-copilot-url" value="${DEFAULT_SIGNALING_URL}" style="width:100%;box-sizing:border-box;margin-bottom:10px;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
@@ -1135,11 +1246,11 @@
</label>
</div>
<div style="margin-bottom:10px;">
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#d1fae5;margin-bottom:4px;"><span id="element2-copilot-selected-audio" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">等待活跃音轨</span><span>原生音轨</span></div>
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#d1fae5;margin-bottom:4px;"><span id="element2-copilot-selected-audio" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">截取给 AI 的音频音量</span><span id="element2-copilot-input-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-input-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#10b981,#fcd34d);transition:width .06s linear;"></div></div>
</div>
<div style="margin-bottom:12px;">
<div style="display:flex;justify-content:space-between;font-size:12px;color:#bfdbfe;margin-bottom:4px;"><span>AI 输出</span><span>业务上行</span></div>
<div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#bfdbfe;margin-bottom:4px;"><span style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">AI 发送给当事人的音频音量</span><span id="element2-copilot-output-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
<div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-output-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#60a5fa,#c084fc);transition:width .06s linear;"></div></div>
</div>
<button id="element2-copilot-toggle" style="width:100%;padding:12px;border:0;border-radius:6px;background:#10b981;color:white;font-weight:700;cursor:pointer;">当前：人工接管｜点击切换 AI</button>