From fe443932f063bc9f2a41a38c52c8a56f7f6324f2 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 9 Jul 2026 08:54:15 +0800 Subject: [PATCH] Refactor AI Element2 Gated Copilot script for improved audio handling - Update the description to clarify that participant audio is fed into AI after both sides are connected. - Introduce new functions to manage audio track patterns and enhance the logic for determining participant and operator presence. - Improve gate status messaging to reflect the online status of participants and operators. - Add volume handling functions for better audio visualization and feedback. - Refactor existing functions to streamline audio processing and improve overall code clarity. --- scripts/copilot-element2-two-party.js | 177 +++++++++++++++++++++----- 1 file changed, 144 insertions(+), 33 deletions(-) diff --git a/scripts/copilot-element2-two-party.js b/scripts/copilot-element2-two-party.js index becea8a..9d381df 100644 --- a/scripts/copilot-element2-two-party.js +++ b/scripts/copilot-element2-two-party.js @@ -2,7 +2,7 @@ // @name 12345 AI Element2 Gated Copilot // @namespace http://tampermonkey.net/ // @version 1.0 -// @description Two-party gated bridge: only feed element-2:audio into AI after both sides are connected. +// @description Two-party gated bridge: only feed participant audio into AI after both sides are connected. // @author You // @match *://yhy-yw.22sj.cn:*/* // @grant none @@ -15,6 +15,7 @@ const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio'; const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video'; + const PARTICIPANT_AUDIO_TRACK_PATTERN = /-audio-3-a$/; const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; const originalAddTrack = RTCPeerConnection.prototype.addTrack; @@ -45,6 +46,11 @@ let aiVideoSender = null; let isAiTakeover = false; let aiAttached = false; + let aiOutputAudioContext = null; + let aiOutputAnalyser = null; + let aiOutputAnalyserData = null; + let aiOutputAnalyserSource = null; + let aiOutputAnalyserTrack = null; let inputVu = 0; let outputVu = 0; let inputHealthTimer = null; @@ -331,40 +337,51 @@ function updateSelectedAudioLabel() { const label = document.getElementById('element2-copilot-selected-audio'); if (!label) return; - const item = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); - label.textContent = item - ? `${audioSourceLabel(item)} ${selectedRemoteAudio.id.slice(0, 8)} score=${Math.round(item.audioScore || 0)}` - : `等待 ${STATIC_AUDIO_ELEMENT_REASON}`; + label.textContent = '截取给 AI 的音频音量'; } - function pageHasBothParties() { + function pageHasParticipant() { const text = document.body?.innerText || ''; - return text.includes('当事人') && text.includes('坐席端'); + return text.includes('当事人'); + } + + function pageHasOperator() { + const text = document.body?.innerText || ''; + return text.includes('坐席端'); + } + + function isParticipantAudioItem(candidate) { + return Boolean( + candidate + && isLive(candidate.track) + && !candidate.track.muted + && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON) + && PARTICIPANT_AUDIO_TRACK_PATTERN.test(candidate.track.id) + ); } function staticAudioItem() { - return Array.from(remoteAudio.values()).find((candidate) => ( - isLive(candidate.track) - && !candidate.track.muted - && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON) - )) || null; + return Array.from(remoteAudio.values()).find(isParticipantAudioItem) || null; } function gateStatus() { const item = staticAudioItem(); + const participantVisible = pageHasParticipant(); + const operatorVisible = pageHasOperator(); return { - bothParties: pageHasBothParties(), - audioReady: Boolean(item), + bothParties: participantVisible && operatorVisible, + participantOnline: participantVisible && Boolean(item), + operatorOnline: operatorVisible, senderReady: Boolean(selectedBusinessSender), track: item?.track || null, }; } function gateMessage(status = gateStatus()) { - if (!status.bothParties) return '等待当事人和坐席端同时在线'; - if (!status.audioReady) return `等待 ${STATIC_AUDIO_ELEMENT_REASON} 有声轨`; - if (!status.senderReady) return '等待业务上行 sender'; - return `就绪：${STATIC_AUDIO_ELEMENT_REASON} -> AI`; + if (!status.operatorOnline) return '等待坐席在线'; + if (!status.participantOnline) return '等待当事人在线'; + if (!status.senderReady) return '等待通话上行就绪'; + return '就绪：可开启 AI 托管'; } function updateGateLabel() { @@ -372,7 +389,7 @@ if (!label) return; const status = gateStatus(); label.textContent = gateMessage(status); - label.style.color = status.bothParties && status.audioReady && status.senderReady ? '#86efac' : '#fcd34d'; + label.style.color = status.participantOnline && status.operatorOnline && status.senderReady ? '#86efac' : '#fcd34d'; } function scoreAudioCandidate(item) { @@ -448,7 +465,7 @@ function selectRemoteAudio(reason) { const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id); - if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted && hasElementReason(current, STATIC_AUDIO_ELEMENT_REASON)) { + if (isParticipantAudioItem(current)) { updateGateLabel(); return selectedRemoteAudio; } @@ -549,17 +566,113 @@ function setVu(id, level) { const el = document.getElementById(id); - if (el) el.style.width = `${Math.min(100, level * 2500)}%`; + if (!el) return; + el.style.width = `${Math.min(100, level * 2500)}%`; + el.style.opacity = level > 0 ? '1' : '0.35'; + } + + function volumePercent(level) { + if (!Number.isFinite(level) || level < 0.0015) return 0; + return Math.round(Math.min(100, level * 2500)); + } + + function volumeDb(level) { + if (!Number.isFinite(level) || level < 0.0015) return '-inf'; + return `${(20 * Math.log10(level)).toFixed(1)}dB`; + } + + function volumeText(level) { + return `vol=${volumePercent(level)}% ${volumeDb(level)}`; + } + + function setVolumeLabel(id, level) { + const el = document.getElementById(id); + if (!el) return; + el.textContent = volumeText(level); } function renderVu() { + refreshAiOutputVuFromAnalyser(); inputVu *= 0.88; outputVu *= 0.88; setVu('element2-copilot-input-vu', inputVu); setVu('element2-copilot-output-vu', outputVu); + setVolumeLabel('element2-copilot-input-volume', inputVu); + setVolumeLabel('element2-copilot-output-volume', outputVu); requestAnimationFrame(renderVu); } + function speechVuLevel(level) { + const noiseFloor = 0.0015; + if (!Number.isFinite(level) || level < noiseFloor) return 0; + return Math.min(0.08, (level - noiseFloor) * 3.2); + } + + function analyserVuLevel(analyser, data) { + if (!analyser || !data) return 0; + analyser.getByteTimeDomainData(data); + let sum = 0; + for (const value of data) { + const centered = (value - 128) / 128; + sum += centered * centered; + } + return speechVuLevel(Math.sqrt(sum / data.length)); + } + + function refreshAiOutputVuFromAnalyser() { + outputVu = Math.max(outputVu, analyserVuLevel(aiOutputAnalyser, aiOutputAnalyserData)); + } + + function createAudioContext() { + const AudioContextCtor = window.AudioContext || window.webkitAudioContext; + return AudioContextCtor ? new AudioContextCtor() : null; + } + + function detachAiOutputMeter() { + try { + aiOutputAnalyserSource?.disconnect(); + } catch {} + aiOutputAnalyser = null; + aiOutputAnalyserData = null; + aiOutputAnalyserSource = null; + aiOutputAnalyserTrack = null; + outputVu = 0; + } + + async function attachAiOutputMeter(track) { + if (!isLive(track)) { + detachAiOutputMeter(); + return false; + } + if (aiOutputAnalyserTrack === track && aiOutputAnalyser) return true; + detachAiOutputMeter(); + if (!aiOutputAudioContext) { + aiOutputAudioContext = createAudioContext(); + if (!aiOutputAudioContext) { + appendChatLine('音频', '当前浏览器不支持 AudioContext，无法显示 AI 输出音量', true); + return false; + } + } + if (aiOutputAudioContext.state === 'suspended') { + await aiOutputAudioContext.resume().catch(() => {}); + } + try { + const stream = new MediaStream([track]); + aiOutputAnalyserSource = aiOutputAudioContext.createMediaStreamSource(stream); + aiOutputAnalyser = aiOutputAudioContext.createAnalyser(); + aiOutputAnalyser.fftSize = 512; + aiOutputAnalyser.smoothingTimeConstant = 0.55; + aiOutputAnalyserData = new Uint8Array(aiOutputAnalyser.fftSize); + aiOutputAnalyserSource.connect(aiOutputAnalyser); + aiOutputAnalyserTrack = track; + return true; + } catch (error) { + log(`failed to attach AI output meter: ${error.message || error}`); + detachAiOutputMeter(); + return false; + } + } + async function refreshStats() { await refreshInputStats(); await refreshOutputStats(); @@ -601,7 +714,7 @@ duration: stat.totalSamplesDuration, }); if (item.track === selectedRemoteAudio) { - inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500), Math.min(0.18, audioLevel)); + inputVu = Math.max(inputVu, speechVuLevel(audioLevel)); } }); } @@ -625,9 +738,6 @@ item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; outboundMemory.set(key, { bytes, packets }); - if (item.sender === selectedBusinessSender) { - outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500)); - } }); } } @@ -846,7 +956,7 @@ selectBusinessSender('connect'); const status = gateStatus(); updateGateLabel(); - if (!status.bothParties || !status.audioReady || !status.senderReady) { + if (!status.operatorOnline || !status.participantOnline || !status.senderReady) { appendChatLine('门槛', gateMessage(status), true); setStatus('disconnected'); return false; @@ -917,6 +1027,7 @@ aiOutputTrackIds.add(event.track.id); remoteAudio.delete(event.track.id); aiAudioTrack = event.track; + attachAiOutputMeter(event.track); log(`received AI audio ${trackLabel(aiAudioTrack)}`, true); aiAudioTrack.onunmute = () => { log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`); @@ -993,6 +1104,7 @@ aiAudioSender = null; aiVideoSender = null; aiAudioTrack = null; + detachAiOutputMeter(); if (inputRecorder?.state === 'recording') stopInputRecording(); if (videoRecorder?.state === 'recording') stopVideoRecording(); setStatus('disconnected'); @@ -1028,7 +1140,8 @@ staticVideoElement: STATIC_VIDEO_ELEMENT_REASON, gate: { bothParties: status.bothParties, - audioReady: status.audioReady, + participantOnline: status.participantOnline, + operatorOnline: status.operatorOnline, senderReady: status.senderReady, track: trackLabel(status.track), }, @@ -1112,13 +1225,11 @@ ].join(';'); panel.innerHTML = `
- AI 元素选轨 + AI 托管 未连接
- 音频输入：${STATIC_AUDIO_ELEMENT_REASON}
- 视频输入：${STATIC_VIDEO_ELEMENT_REASON}
- 等待当事人和坐席端同时在线 + 等待当事人在线
@@ -1135,11 +1246,11 @@
-
等待活跃音轨原生音轨
+
截取给 AI 的音频音量vol=0% -inf
-
AI 输出业务上行
+
AI 发送给当事人的音频音量vol=0% -inf