Refactor AI Element2 Gated Copilot script for improved audio handling

- Update the description to clarify that participant audio is fed into AI after both sides are connected.
- Introduce new functions to manage audio track patterns and enhance the logic for determining participant and operator presence.
- Improve gate status messaging to reflect the online status of participants and operators.
- Add volume handling functions for better audio visualization and feedback.
- Refactor existing functions to streamline audio processing and improve overall code clarity.

scripts/copilot-element2-two-party.js

@@ -2,7 +2,7 @@
 // @name         12345 AI Element2 Gated Copilot
 // @namespace    http://tampermonkey.net/
 // @version      1.0
-// @description  Two-party gated bridge: only feed element-2:audio into AI after both sides are connected.
+// @description  Two-party gated bridge: only feed participant audio into AI after both sides are connected.
 // @author       You
 // @match        *://yhy-yw.22sj.cn:*/*
 // @grant        none
@@ -15,6 +15,7 @@
     const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice';
     const STATIC_AUDIO_ELEMENT_REASON = 'element-2:audio';
     const STATIC_VIDEO_ELEMENT_REASON = 'element-0:video';
+    const PARTICIPANT_AUDIO_TRACK_PATTERN = /-audio-3-a$/;
 
     const originalAddEventListener = RTCPeerConnection.prototype.addEventListener;
     const originalAddTrack = RTCPeerConnection.prototype.addTrack;
@@ -45,6 +46,11 @@
     let aiVideoSender = null;
     let isAiTakeover = false;
     let aiAttached = false;
+    let aiOutputAudioContext = null;
+    let aiOutputAnalyser = null;
+    let aiOutputAnalyserData = null;
+    let aiOutputAnalyserSource = null;
+    let aiOutputAnalyserTrack = null;
     let inputVu = 0;
     let outputVu = 0;
     let inputHealthTimer = null;
@@ -331,40 +337,51 @@
     function updateSelectedAudioLabel() {
         const label = document.getElementById('element2-copilot-selected-audio');
         if (!label) return;
-        const item = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id);
+        label.textContent = '截取给 AI 的音频音量';
-        label.textContent = item
-            ? `${audioSourceLabel(item)} ${selectedRemoteAudio.id.slice(0, 8)} score=${Math.round(item.audioScore || 0)}`
-            : `等待 ${STATIC_AUDIO_ELEMENT_REASON}`;
     }
 
-    function pageHasBothParties() {
+    function pageHasParticipant() {
         const text = document.body?.innerText || '';
-        return text.includes('当事人') && text.includes('坐席端');
+        return text.includes('当事人');
+    }
+
+    function pageHasOperator() {
+        const text = document.body?.innerText || '';
+        return text.includes('坐席端');
+    }
+
+    function isParticipantAudioItem(candidate) {
+        return Boolean(
+            candidate
+            && isLive(candidate.track)
+            && !candidate.track.muted
+            && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON)
+            && PARTICIPANT_AUDIO_TRACK_PATTERN.test(candidate.track.id)
+        );
     }
 
     function staticAudioItem() {
-        return Array.from(remoteAudio.values()).find((candidate) => (
+        return Array.from(remoteAudio.values()).find(isParticipantAudioItem) || null;
-            isLive(candidate.track)
-            && !candidate.track.muted
-            && hasElementReason(candidate, STATIC_AUDIO_ELEMENT_REASON)
-        )) || null;
     }
 
     function gateStatus() {
         const item = staticAudioItem();
+        const participantVisible = pageHasParticipant();
+        const operatorVisible = pageHasOperator();
         return {
-            bothParties: pageHasBothParties(),
+            bothParties: participantVisible && operatorVisible,
-            audioReady: Boolean(item),
+            participantOnline: participantVisible && Boolean(item),
+            operatorOnline: operatorVisible,
             senderReady: Boolean(selectedBusinessSender),
             track: item?.track || null,
         };
     }
 
     function gateMessage(status = gateStatus()) {
-        if (!status.bothParties) return '等待当事人和坐席端同时在线';
+        if (!status.operatorOnline) return '等待坐席在线';
-        if (!status.audioReady) return `等待 ${STATIC_AUDIO_ELEMENT_REASON} 有声轨`;
+        if (!status.participantOnline) return '等待当事人在线';
-        if (!status.senderReady) return '等待业务上行 sender';
+        if (!status.senderReady) return '等待通话上行就绪';
-        return `就绪：${STATIC_AUDIO_ELEMENT_REASON} -> AI`;
+        return '就绪：可开启 AI 托管';
     }
 
     function updateGateLabel() {
@@ -372,7 +389,7 @@
         if (!label) return;
         const status = gateStatus();
         label.textContent = gateMessage(status);
-        label.style.color = status.bothParties && status.audioReady && status.senderReady ? '#86efac' : '#fcd34d';
+        label.style.color = status.participantOnline && status.operatorOnline && status.senderReady ? '#86efac' : '#fcd34d';
     }
 
     function scoreAudioCandidate(item) {
@@ -448,7 +465,7 @@
 
     function selectRemoteAudio(reason) {
         const current = selectedRemoteAudio && remoteAudio.get(selectedRemoteAudio.id);
-        if (isLive(selectedRemoteAudio) && !selectedRemoteAudio.muted && hasElementReason(current, STATIC_AUDIO_ELEMENT_REASON)) {
+        if (isParticipantAudioItem(current)) {
             updateGateLabel();
             return selectedRemoteAudio;
         }
@@ -549,17 +566,113 @@
 
     function setVu(id, level) {
         const el = document.getElementById(id);
-        if (el) el.style.width = `${Math.min(100, level * 2500)}%`;
+        if (!el) return;
+        el.style.width = `${Math.min(100, level * 2500)}%`;
+        el.style.opacity = level > 0 ? '1' : '0.35';
+    }
+
+    function volumePercent(level) {
+        if (!Number.isFinite(level) || level < 0.0015) return 0;
+        return Math.round(Math.min(100, level * 2500));
+    }
+
+    function volumeDb(level) {
+        if (!Number.isFinite(level) || level < 0.0015) return '-inf';
+        return `${(20 * Math.log10(level)).toFixed(1)}dB`;
+    }
+
+    function volumeText(level) {
+        return `vol=${volumePercent(level)}% ${volumeDb(level)}`;
+    }
+
+    function setVolumeLabel(id, level) {
+        const el = document.getElementById(id);
+        if (!el) return;
+        el.textContent = volumeText(level);
     }
 
     function renderVu() {
+        refreshAiOutputVuFromAnalyser();
         inputVu *= 0.88;
         outputVu *= 0.88;
         setVu('element2-copilot-input-vu', inputVu);
         setVu('element2-copilot-output-vu', outputVu);
+        setVolumeLabel('element2-copilot-input-volume', inputVu);
+        setVolumeLabel('element2-copilot-output-volume', outputVu);
         requestAnimationFrame(renderVu);
     }
 
+    function speechVuLevel(level) {
+        const noiseFloor = 0.0015;
+        if (!Number.isFinite(level) || level < noiseFloor) return 0;
+        return Math.min(0.08, (level - noiseFloor) * 3.2);
+    }
+
+    function analyserVuLevel(analyser, data) {
+        if (!analyser || !data) return 0;
+        analyser.getByteTimeDomainData(data);
+        let sum = 0;
+        for (const value of data) {
+            const centered = (value - 128) / 128;
+            sum += centered * centered;
+        }
+        return speechVuLevel(Math.sqrt(sum / data.length));
+    }
+
+    function refreshAiOutputVuFromAnalyser() {
+        outputVu = Math.max(outputVu, analyserVuLevel(aiOutputAnalyser, aiOutputAnalyserData));
+    }
+
+    function createAudioContext() {
+        const AudioContextCtor = window.AudioContext || window.webkitAudioContext;
+        return AudioContextCtor ? new AudioContextCtor() : null;
+    }
+
+    function detachAiOutputMeter() {
+        try {
+            aiOutputAnalyserSource?.disconnect();
+        } catch {}
+        aiOutputAnalyser = null;
+        aiOutputAnalyserData = null;
+        aiOutputAnalyserSource = null;
+        aiOutputAnalyserTrack = null;
+        outputVu = 0;
+    }
+
+    async function attachAiOutputMeter(track) {
+        if (!isLive(track)) {
+            detachAiOutputMeter();
+            return false;
+        }
+        if (aiOutputAnalyserTrack === track && aiOutputAnalyser) return true;
+        detachAiOutputMeter();
+        if (!aiOutputAudioContext) {
+            aiOutputAudioContext = createAudioContext();
+            if (!aiOutputAudioContext) {
+                appendChatLine('音频', '当前浏览器不支持 AudioContext，无法显示 AI 输出音量', true);
+                return false;
+            }
+        }
+        if (aiOutputAudioContext.state === 'suspended') {
+            await aiOutputAudioContext.resume().catch(() => {});
+        }
+        try {
+            const stream = new MediaStream([track]);
+            aiOutputAnalyserSource = aiOutputAudioContext.createMediaStreamSource(stream);
+            aiOutputAnalyser = aiOutputAudioContext.createAnalyser();
+            aiOutputAnalyser.fftSize = 512;
+            aiOutputAnalyser.smoothingTimeConstant = 0.55;
+            aiOutputAnalyserData = new Uint8Array(aiOutputAnalyser.fftSize);
+            aiOutputAnalyserSource.connect(aiOutputAnalyser);
+            aiOutputAnalyserTrack = track;
+            return true;
+        } catch (error) {
+            log(`failed to attach AI output meter: ${error.message || error}`);
+            detachAiOutputMeter();
+            return false;
+        }
+    }
+
     async function refreshStats() {
         await refreshInputStats();
         await refreshOutputStats();
@@ -601,7 +714,7 @@
                     duration: stat.totalSamplesDuration,
                 });
                 if (item.track === selectedRemoteAudio) {
-                    inputVu = Math.max(inputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500), Math.min(0.18, audioLevel));
+                    inputVu = Math.max(inputVu, speechVuLevel(audioLevel));
                 }
             });
         }
@@ -625,9 +738,6 @@
                 item.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
                 item.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
                 outboundMemory.set(key, { bytes, packets });
-                if (item.sender === selectedBusinessSender) {
-                    outputVu = Math.max(outputVu, Math.min(0.12, item.bytesDelta / 12000), Math.min(0.04, item.packetsDelta / 500));
-                }
             });
         }
     }
@@ -846,7 +956,7 @@
         selectBusinessSender('connect');
         const status = gateStatus();
         updateGateLabel();
-        if (!status.bothParties || !status.audioReady || !status.senderReady) {
+        if (!status.operatorOnline || !status.participantOnline || !status.senderReady) {
             appendChatLine('门槛', gateMessage(status), true);
             setStatus('disconnected');
             return false;
@@ -917,6 +1027,7 @@
             aiOutputTrackIds.add(event.track.id);
             remoteAudio.delete(event.track.id);
             aiAudioTrack = event.track;
+            attachAiOutputMeter(event.track);
             log(`received AI audio ${trackLabel(aiAudioTrack)}`, true);
             aiAudioTrack.onunmute = () => {
                 log(`AI audio unmuted ${trackLabel(aiAudioTrack)}`);
@@ -993,6 +1104,7 @@
         aiAudioSender = null;
         aiVideoSender = null;
         aiAudioTrack = null;
+        detachAiOutputMeter();
         if (inputRecorder?.state === 'recording') stopInputRecording();
         if (videoRecorder?.state === 'recording') stopVideoRecording();
         setStatus('disconnected');
@@ -1028,7 +1140,8 @@
             staticVideoElement: STATIC_VIDEO_ELEMENT_REASON,
             gate: {
                 bothParties: status.bothParties,
-                audioReady: status.audioReady,
+                participantOnline: status.participantOnline,
+                operatorOnline: status.operatorOnline,
                 senderReady: status.senderReady,
                 track: trackLabel(status.track),
             },
@@ -1112,13 +1225,11 @@
         ].join(';');
         panel.innerHTML = `
             <div style="display:flex;justify-content:space-between;gap:12px;align-items:center;margin-bottom:12px;">
-                <strong style="font-size:15px;color:#f9a8d4;">AI 元素选轨</strong>
+                <strong style="font-size:15px;color:#f9a8d4;">AI 托管</strong>
                 <span id="element2-copilot-status" style="font-size:12px;color:#d1d5db;">未连接</span>
             </div>
             <div style="font-size:11px;color:#9ca3af;margin-bottom:10px;line-height:1.5;">
-                音频输入：${STATIC_AUDIO_ELEMENT_REASON}<br>
+                <span id="element2-copilot-gate" style="color:#fcd34d;">等待当事人在线</span>
-                视频输入：${STATIC_VIDEO_ELEMENT_REASON}<br>
-                <span id="element2-copilot-gate" style="color:#fcd34d;">等待当事人和坐席端同时在线</span>
             </div>
             <label style="display:block;font-size:12px;color:#9ca3af;margin-bottom:4px;">信令地址</label>
             <input id="element2-copilot-url" value="${DEFAULT_SIGNALING_URL}" style="width:100%;box-sizing:border-box;margin-bottom:10px;padding:8px;border:1px solid #4b5563;border-radius:6px;background:#111827;color:white;">
@@ -1135,11 +1246,11 @@
                 </label>
             </div>
             <div style="margin-bottom:10px;">
-                <div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#d1fae5;margin-bottom:4px;"><span id="element2-copilot-selected-audio" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">等待活跃音轨</span><span>原生音轨</span></div>
+                <div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#d1fae5;margin-bottom:4px;"><span id="element2-copilot-selected-audio" style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">截取给 AI 的音频音量</span><span id="element2-copilot-input-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
                 <div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-input-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#10b981,#fcd34d);transition:width .06s linear;"></div></div>
             </div>
             <div style="margin-bottom:12px;">
-                <div style="display:flex;justify-content:space-between;font-size:12px;color:#bfdbfe;margin-bottom:4px;"><span>AI 输出</span><span>业务上行</span></div>
+                <div style="display:flex;justify-content:space-between;gap:8px;font-size:12px;color:#bfdbfe;margin-bottom:4px;"><span style="overflow:hidden;text-overflow:ellipsis;white-space:nowrap;">AI 发送给当事人的音频音量</span><span id="element2-copilot-output-volume" style="flex:0 0 auto;">vol=0% -inf</span></div>
                 <div style="height:8px;border-radius:999px;overflow:hidden;background:#374151;"><div id="element2-copilot-output-vu" style="height:100%;width:0%;background:linear-gradient(90deg,#60a5fa,#c084fc);transition:width .06s linear;"></div></div>
             </div>
             <button id="element2-copilot-toggle" style="width:100%;padding:12px;border:0;border-radius:6px;background:#10b981;color:white;font-weight:700;cursor:pointer;">当前：人工接管｜点击切换 AI</button>