From 03478fa557901136fe8ff532e047c414da0aa880 Mon Sep 17 00:00:00 2001 From: Xin Wang Date: Thu, 9 Jul 2026 13:33:02 +0800 Subject: [PATCH] Add Audio Sender Sine Tester script for WebRTC audio management - Introduce a new user script `audio-sender-sine-tester.js` that lists current WebRTC audio senders and allows temporary sine wave transmission through selected senders. - Implement functionality for managing audio sender candidates, tracking audio statistics, and providing real-time logging of audio activities. - Enhance audio testing capabilities with sine wave generation and track replacement features, improving overall audio management in the WebRTC environment. - This addition supports better monitoring and testing of audio streams in applications utilizing WebRTC. --- scripts/audio-sender-sine-tester.js | 371 +++++ scripts/copilot-native-direct-output.js | 1725 ----------------------- scripts/copilot-native.js | 1542 -------------------- scripts/copilot.js | 311 +++- 4 files changed, 640 insertions(+), 3309 deletions(-) create mode 100644 scripts/audio-sender-sine-tester.js delete mode 100644 scripts/copilot-native-direct-output.js delete mode 100644 scripts/copilot-native.js diff --git a/scripts/audio-sender-sine-tester.js b/scripts/audio-sender-sine-tester.js new file mode 100644 index 0000000..b6f6bed --- /dev/null +++ b/scripts/audio-sender-sine-tester.js @@ -0,0 +1,371 @@ +// ==UserScript== +// @name 12345 Audio Sender Sine Tester +// @namespace http://tampermonkey.net/ +// @version 1.0 +// @description List current WebRTC audio senders and temporarily send a sine wave through a selected sender. +// @author You +// @match *://yhy-yw.22sj.cn:*/* +// @grant none +// @run-at document-start +// ==/UserScript== + +(function () { + 'use strict'; + + const originalAddTrack = RTCPeerConnection.prototype.addTrack; + const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; + const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; + + const senderCandidates = new Map(); + const outboundStatsMemory = new Map(); + + let activeTest = null; + let lastRenderAt = 0; + + function nowText() { + return new Date().toLocaleTimeString([], { hour12: false }); + } + + function shortId(id) { + if (!id) return 'null'; + return id.length <= 18 ? id : `${id.slice(0, 8)}...${id.slice(-6)}`; + } + + function log(message) { + console.log(`[Audio Sine Tester] ${message}`); + const box = document.getElementById('audio-sine-tester-log'); + if (!box) return; + const row = document.createElement('div'); + row.textContent = `[${nowText()}] ${message}`; + box.appendChild(row); + while (box.children.length > 80) box.removeChild(box.firstChild); + box.scrollTop = box.scrollHeight; + } + + function ensureSenderCandidate(sender, track, source) { + if (!sender) return null; + const senderTrack = track || sender.track; + if (senderTrack && senderTrack.kind !== 'audio') return null; + + const id = sender.__audioSineTesterId || `sender-${senderCandidates.size + 1}`; + sender.__audioSineTesterId = id; + + let candidate = senderCandidates.get(id); + if (!candidate) { + candidate = { + id, + sender, + track: senderTrack || null, + originalTrack: senderTrack || null, + sources: new Set(), + bytesSent: 0, + packetsSent: 0, + bytesDelta: 0, + packetsDelta: 0, + score: 0, + }; + senderCandidates.set(id, candidate); + log(`found audio sender ${id} (${source}) track=${senderTrack?.id || 'null'}`); + } + + candidate.sender = sender; + candidate.track = sender.track || senderTrack || candidate.track; + if (sender.track && sender.track.kind === 'audio' && !sender.track.__audioSineTesterTrack) { + candidate.originalTrack = sender.track; + } + candidate.sources.add(source); + wrapSender(sender); + return candidate; + } + + function wrapSender(sender) { + if (!sender || sender.__audioSineTesterWrapped) return; + const originalReplaceTrack = sender.replaceTrack?.bind(sender); + if (!originalReplaceTrack) return; + + sender.replaceTrack = async function (track) { + const result = await originalReplaceTrack(track); + const candidate = ensureSenderCandidate(sender, track, 'replaceTrack'); + if (candidate) { + candidate.track = track || sender.track || candidate.track; + if (track && !track.__audioSineTesterTrack) candidate.originalTrack = track; + log(`${candidate.id} replaceTrack -> ${track?.id || 'null'}`); + } + return result; + }; + sender.__audioSineTesterWrapped = true; + } + + async function refreshSenderStats() { + for (const candidate of senderCandidates.values()) { + const sender = candidate.sender; + candidate.track = sender.track || candidate.track; + if (!sender.getStats) continue; + + try { + const report = await sender.getStats(); + report.forEach((stat) => { + const kind = stat.kind || stat.mediaType; + if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; + + const key = `${candidate.id}:${stat.id}`; + const prev = outboundStatsMemory.get(key); + const bytes = stat.bytesSent || 0; + const packets = stat.packetsSent || 0; + candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; + candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; + candidate.bytesSent = bytes; + candidate.packetsSent = packets; + outboundStatsMemory.set(key, { bytes, packets }); + candidate.score = scoreCandidate(candidate); + }); + } catch (error) { + candidate.sources.add(`stats-error:${error.message || error}`); + } + } + } + + function scoreCandidate(candidate) { + let score = 0; + if (candidate.track?.kind === 'audio') score += 1000; + if (candidate.track?.readyState === 'live') score += 800; + if (candidate.sources.has('pc-addTrack')) score += 1500; + if (candidate.bytesSent > 0) score += 600; + if (candidate.bytesDelta > 0) score += Math.min(2500, candidate.bytesDelta * 4); + if (candidate.packetsDelta > 0) score += Math.min(1000, candidate.packetsDelta * 12); + return score; + } + + function makeSineTrack(frequency) { + const AudioCtx = window.AudioContext || window.webkitAudioContext; + const context = new AudioCtx({ sampleRate: 48000 }); + const oscillator = context.createOscillator(); + const gain = context.createGain(); + const dest = context.createMediaStreamDestination(); + + oscillator.type = 'sine'; + oscillator.frequency.value = frequency; + gain.gain.value = 0.08; + oscillator.connect(gain); + gain.connect(dest); + oscillator.start(); + + const track = dest.stream.getAudioTracks()[0]; + track.__audioSineTesterTrack = true; + track.__audioSineTesterContext = context; + track.__audioSineTesterOscillator = oscillator; + track.__audioSineTesterGain = gain; + return track; + } + + function stopTrack(track) { + if (!track) return; + try { track.__audioSineTesterOscillator?.stop(); } catch {} + try { track.stop(); } catch {} + try { track.__audioSineTesterContext?.close(); } catch {} + } + + async function startSine(candidateId, frequency) { + const candidate = senderCandidates.get(candidateId); + if (!candidate?.sender) return; + + await stopSine(false); + + const originalTrack = candidate.sender.track || candidate.originalTrack || null; + const sineTrack = makeSineTrack(frequency); + await candidate.sender.replaceTrack(sineTrack); + + activeTest = { + candidateId, + sender: candidate.sender, + originalTrack, + sineTrack, + frequency, + startedAt: Date.now(), + }; + + candidate.originalTrack = originalTrack; + candidate.track = sineTrack; + log(`START ${frequency}Hz on ${candidateId}; original=${originalTrack?.id || 'null'}`); + renderPanel(); + } + + async function stopSine(shouldLog = true) { + if (!activeTest) return; + const test = activeTest; + activeTest = null; + + try { + await test.sender.replaceTrack(test.originalTrack || null); + if (shouldLog) log(`STOP ${test.frequency}Hz on ${test.candidateId}; restored=${test.originalTrack?.id || 'null'}`); + } catch (error) { + log(`restore failed on ${test.candidateId}: ${error.message || error}`); + } finally { + stopTrack(test.sineTrack); + } + renderPanel(); + } + + function dump() { + return { + activeTest: activeTest ? { + candidateId: activeTest.candidateId, + frequency: activeTest.frequency, + sineTrackId: activeTest.sineTrack.id, + originalTrackId: activeTest.originalTrack?.id || null, + seconds: Math.round((Date.now() - activeTest.startedAt) / 1000), + } : null, + senders: Array.from(senderCandidates.values()) + .sort((a, b) => b.score - a.score) + .map((candidate) => ({ + id: candidate.id, + active: activeTest?.candidateId === candidate.id, + trackId: candidate.sender.track?.id || candidate.track?.id || null, + trackLabel: candidate.sender.track?.label || candidate.track?.label || null, + originalTrackId: candidate.originalTrack?.id || null, + trackEnabled: candidate.sender.track?.enabled ?? null, + trackMuted: candidate.sender.track?.muted ?? null, + trackReadyState: candidate.sender.track?.readyState || null, + bytesSent: candidate.bytesSent, + packetsSent: candidate.packetsSent, + bytesDelta: candidate.bytesDelta, + packetsDelta: candidate.packetsDelta, + score: Math.round(candidate.score), + sources: Array.from(candidate.sources), + })), + }; + } + + function renderPanel() { + if (!document.body) return; + let panel = document.getElementById('audio-sine-tester-panel'); + if (!panel) { + panel = document.createElement('div'); + panel.id = 'audio-sine-tester-panel'; + panel.style.cssText = [ + 'position:fixed', + 'top:20px', + 'right:20px', + 'width:500px', + 'max-height:76vh', + 'z-index:999999', + 'box-sizing:border-box', + 'padding:12px', + 'border-radius:8px', + 'background:rgba(3,7,18,.96)', + 'color:#e5e7eb', + 'font:12px ui-monospace,Menlo,monospace', + 'box-shadow:0 12px 32px rgba(0,0,0,.35)', + 'overflow:auto', + ].join(';'); + panel.innerHTML = ` +
+ Audio Sender Sine Tester + +
+
+
+
+ +
+
+ `; + document.body.appendChild(panel); + + document.getElementById('audio-sine-copy').addEventListener('click', async () => { + const text = JSON.stringify(window.__audioSineTesterDump(), null, 2); + await navigator.clipboard?.writeText(text).catch(() => {}); + console.log('[Audio Sine Tester dump]', text); + log('dump copied to clipboard and console'); + }); + document.getElementById('audio-sine-stop').addEventListener('click', () => stopSine()); + } + + const data = dump(); + document.getElementById('audio-sine-summary').textContent = + `senders=${data.senders.length}, active=${data.activeTest ? `${data.activeTest.candidateId} ${data.activeTest.frequency}Hz ${data.activeTest.seconds}s` : 'none'}, time=${nowText()}`; + + document.getElementById('audio-sine-list').innerHTML = data.senders.map((row) => { + const pct = Math.min(100, Math.max(row.bytesDelta / 20, row.packetsDelta * 2)); + const active = row.active; + return ` +
+
+ ${active ? '*' : '-'} ${row.id} + score=${row.score} +
+
+
+
+
+ track=${shortId(row.trackId)} label=${row.trackLabel || 'null'} muted=${row.trackMuted} bytesDelta=${row.bytesDelta} packetsDelta=${row.packetsDelta} +
+
+ + +
+
${row.sources.join(', ')}
+
+ `; + }).join(''); + + panel.querySelectorAll('[data-sine]').forEach((button) => { + button.onclick = () => { + const [id, hz] = button.dataset.sine.split(':'); + startSine(id, Number(hz)); + }; + }); + } + + RTCPeerConnection.prototype.addTrack = function (track, ...streams) { + const sender = originalAddTrack.call(this, track, ...streams); + if (track?.kind === 'audio') ensureSenderCandidate(sender, track, 'pc-addTrack'); + return sender; + }; + + if (originalAddTransceiver) { + RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { + const transceiver = originalAddTransceiver.call(this, trackOrKind, init); + const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; + if (kind === 'audio') { + const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; + ensureSenderCandidate(transceiver.sender, track, 'pc-addTransceiver'); + } + return transceiver; + }; + } + + RTCPeerConnection.prototype.setRemoteDescription = function () { + const result = originalSetRemoteDescription.apply(this, arguments); + Promise.resolve(result).then(() => { + try { + for (const sender of this.getSenders?.() || []) { + ensureSenderCandidate(sender, sender.track, 'sender-after-srd'); + } + } catch (error) { + console.warn('[Audio Sine Tester] failed to scan senders', error); + } + }); + return result; + }; + + window.__audioSineTesterDump = dump; + window.__audioSineTesterStop = stopSine; + + setInterval(refreshSenderStats, 250); + setInterval(() => { + if (Date.now() - lastRenderAt < 500) return; + lastRenderAt = Date.now(); + renderPanel(); + }, 250); + + window.addEventListener('beforeunload', () => { + if (activeTest) stopSine(false); + }); + + if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', renderPanel, { once: true }); + } else { + renderPanel(); + } +})(); diff --git a/scripts/copilot-native-direct-output.js b/scripts/copilot-native-direct-output.js deleted file mode 100644 index 0545c3a..0000000 --- a/scripts/copilot-native-direct-output.js +++ /dev/null @@ -1,1725 +0,0 @@ -// ==UserScript== -// @name 12345 AI Native Track Copilot Direct Output 2 -// @namespace http://tampermonkey.net/ -// @version 1.0 -// @description Native-track bridge with direct AI response track output to business sender. -// @author You -// @match *://yhy-yw.22sj.cn:*/* -// @grant none -// @run-at document-start -// ==/UserScript== - -(function () { - 'use strict'; - - const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; - const AUDIO_UPLINK_STORAGE_KEY = 'native-copilot-send-audio'; - const VIDEO_UPLINK_STORAGE_KEY = 'native-copilot-send-video'; - - const originalGetUserMedia = navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices); - const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; - const originalAddTrack = RTCPeerConnection.prototype.addTrack; - const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; - const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; - const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); - - const remoteStreams = new Set(); - const ignoredTrackIds = new Set(); - const audioCandidates = new Map(); - const videoCandidates = new Map(); - const businessSenderCandidates = new Map(); - const localCaptureTrackIds = new Set(); - window.__aiNativeCopilotReceivers = window.__aiNativeCopilotReceivers || new Set(); - - let aiPc = null; - let aiWs = null; - let aiDc = null; - let aiAudioSender = null; - let aiVideoSender = null; - - let businessAudioSender = null; - let businessOriginalAudioTrack = null; - let uplinkAudioContext = null; - let uplinkDestination = null; - let uplinkStableTrack = null; - let manualSource = null; - let manualGain = null; - let aiSource = null; - let aiGain = null; - let manualSourceTrack = null; - let aiSourceTrack = null; - - let remoteUserAudioTrack = null; - let remoteUserVideoTrack = null; - let aiResponseAudioTrack = null; - let remoteUserAudioLocked = false; - let remoteUserVideoLocked = false; - - let isAiTakeover = false; - let aiTrackAttachedToBusiness = false; - let inputVuLevel = 0; - let aiOutVuLevel = 0; - let aiInputHealthTimer = null; - let aiOutputHealthTimer = null; - let assistantLine = null; - let assistantBuffer = ''; - let inputRecorder = null; - let inputRecordingChunks = []; - let inputRecordingTrack = null; - let inputRecordingStartedAt = 0; - let videoRecorder = null; - let videoRecordingChunks = []; - let videoRecordingTrack = null; - let videoRecordingStartedAt = 0; - - const receiverStatsMemory = new Map(); - const senderStatsMemory = new Map(); - const inboundStatsMemory = new Map(); - const outboundStatsMemory = new Map(); - const businessOutboundStats = { - bytesSent: 0, - packetsSent: 0, - bytesDelta: 0, - packetsDelta: 0, - }; - - function log(message) { - console.log(`[AI Native Copilot] ${message}`); - } - - function readBoolSetting(key, fallback) { - try { - const value = localStorage.getItem(key); - if (value === null) return fallback; - return value === 'true'; - } catch { - return fallback; - } - } - - function writeBoolSetting(key, value) { - try { - localStorage.setItem(key, value ? 'true' : 'false'); - } catch { - // Keep the live checkbox state even if storage is unavailable. - } - } - - function shouldSendAudioToAi() { - return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); - } - - function shouldSendVideoToAi() { - return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); - } - - function appendChatLine(role, text, important = false) { - const box = document.getElementById('native-copilot-log'); - if (!box) return; - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - - const row = document.createElement('div'); - row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; - row.style.marginBottom = '4px'; - if (important) row.style.color = '#fcd34d'; - box.appendChild(row); - while (box.children.length > 120) box.removeChild(box.firstChild); - box.scrollTop = box.scrollHeight; - return row; - } - - function startAssistantLine() { - assistantBuffer = ''; - assistantLine = null; - } - - function appendAssistantDelta(delta) { - if (!delta) return; - const box = document.getElementById('native-copilot-log'); - if (!box) { - console.log(`[AI Native Copilot] AI: ${delta}`); - return; - } - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - if (!assistantLine) assistantLine = appendChatLine('AI', ''); - assistantBuffer += delta; - assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; - box.scrollTop = box.scrollHeight; - } - - function finishAssistantLine() { - assistantLine = null; - assistantBuffer = ''; - } - - function logUserTranscript(text) { - if (!text?.trim()) return; - finishAssistantLine(); - appendChatLine('用户', text.trim(), true); - } - - function recorderMimeType() { - if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; - if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; - return ''; - } - - function recordingFileName(track) { - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; - return `native-ai-input-${trackPart}-${stamp}.webm`; - } - - function videoRecordingFileName(track) { - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; - return `native-ai-video-${trackPart}-${stamp}.webm`; - } - - function updateRecordButton() { - const button = document.getElementById('native-copilot-record'); - if (!button) return; - const recording = inputRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; - button.style.background = recording ? '#dc2626' : '#2563eb'; - } - - function videoRecorderMimeType() { - if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8'; - if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm'; - return ''; - } - - function updateVideoRecordButton() { - const button = document.getElementById('native-copilot-record-video'); - if (!button) return; - const recording = videoRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频'; - button.style.background = recording ? '#dc2626' : '#475569'; - } - - async function startInputRecording() { - if (inputRecorder?.state === 'recording') return; - chooseRemoteAudioTrack('record'); - if (!isUsableTrack(remoteUserAudioTrack)) { - appendChatLine('录音', '没有可录制的 AI 输入音轨', true); - return; - } - if (!window.MediaRecorder) { - appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); - return; - } - - inputRecordingChunks = []; - inputRecordingTrack = remoteUserAudioTrack; - inputRecordingStartedAt = Date.now(); - const stream = new MediaStream([inputRecordingTrack]); - const mimeType = recorderMimeType(); - inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); - inputRecorder.ondataavailable = (event) => { - if (event.data?.size > 0) inputRecordingChunks.push(event.data); - }; - inputRecorder.onstop = () => { - const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - link.download = recordingFileName(inputRecordingTrack); - document.body.appendChild(link); - link.click(); - link.remove(); - setTimeout(() => URL.revokeObjectURL(url), 30000); - - const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); - appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); - inputRecorder = null; - inputRecordingChunks = []; - inputRecordingTrack = null; - updateRecordButton(); - }; - inputRecorder.start(250); - appendChatLine('录音', `开始录制 AI 输入：${describeTrack(inputRecordingTrack)}`, true); - updateRecordButton(); - } - - function stopInputRecording() { - if (inputRecorder?.state === 'recording') inputRecorder.stop(); - } - - function toggleInputRecording() { - if (inputRecorder?.state === 'recording') stopInputRecording(); - else startInputRecording(); - } - - async function startVideoRecording() { - if (videoRecorder?.state === 'recording') return; - chooseRemoteVideoTrack('record-video'); - if (!remoteUserVideoLocked || !isUsableTrack(remoteUserVideoTrack)) { - appendChatLine('录制视频', '没有可录制的上半部用户视频轨', true); - return; - } - if (!window.MediaRecorder) { - appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true); - return; - } - - videoRecordingChunks = []; - videoRecordingTrack = remoteUserVideoTrack; - videoRecordingStartedAt = Date.now(); - const stream = new MediaStream([videoRecordingTrack]); - const mimeType = videoRecorderMimeType(); - videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); - videoRecorder.ondataavailable = (event) => { - if (event.data?.size > 0) videoRecordingChunks.push(event.data); - }; - videoRecorder.onstop = () => { - const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' }); - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - link.download = videoRecordingFileName(videoRecordingTrack); - document.body.appendChild(link); - link.click(); - link.remove(); - setTimeout(() => URL.revokeObjectURL(url), 30000); - - const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1); - appendChatLine('录制视频', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); - videoRecorder = null; - videoRecordingChunks = []; - videoRecordingTrack = null; - updateVideoRecordButton(); - }; - videoRecorder.start(250); - appendChatLine('录制视频', `开始录制 AI 视频：${describeTrack(videoRecordingTrack)}`, true); - updateVideoRecordButton(); - } - - function stopVideoRecording() { - if (videoRecorder?.state === 'recording') videoRecorder.stop(); - } - - function toggleVideoRecording() { - if (videoRecorder?.state === 'recording') stopVideoRecording(); - else startVideoRecording(); - } - - function generatePcId() { - const bytes = new Uint8Array(16); - crypto.getRandomValues(bytes); - return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); - } - - function isInternalPc(pc) { - return Boolean(pc?.__aiNativeCopilotInternal); - } - - function isUsableTrack(track) { - return track && track.readyState === 'live' && !ignoredTrackIds.has(track.id); - } - - function isLiveTrack(track) { - return track && track.readyState === 'live'; - } - - function candidateFor(track, candidates) { - let candidate = candidates.get(track.id); - if (!candidate) { - candidate = { - track, - receiver: null, - reasons: new Set(), - elements: new Set(), - firstSeenAt: Date.now(), - lastSeenAt: 0, - lastPacketAt: 0, - bytesReceived: 0, - packetsReceived: 0, - bytesDelta: 0, - packetsDelta: 0, - level: 0, - score: 0, - logged: false, - }; - candidates.set(track.id, candidate); - } - candidate.track = track; - candidate.lastSeenAt = Date.now(); - return candidate; - } - - function businessSenderCandidateFor(sender, track, reason) { - if (!sender) return null; - const senderTrack = track || sender.track; - if (senderTrack && senderTrack.kind !== 'audio') return null; - - const id = sender.__aiNativeCopilotBusinessSenderId || `business-sender-${businessSenderCandidates.size + 1}`; - sender.__aiNativeCopilotBusinessSenderId = id; - - let candidate = businessSenderCandidates.get(id); - if (!candidate) { - candidate = { - id, - sender, - track: senderTrack || null, - originalTrack: senderTrack || null, - sources: new Set(), - bytesSent: 0, - packetsSent: 0, - bytesDelta: 0, - packetsDelta: 0, - score: 0, - logged: false, - }; - businessSenderCandidates.set(id, candidate); - } - - candidate.sender = sender; - candidate.track = senderTrack || sender.track || candidate.track; - if (!isAiTakeover && senderTrack && senderTrack !== aiResponseAudioTrack && !isUplinkMixerTrack(senderTrack)) { - candidate.originalTrack = senderTrack; - } - candidate.sources.add(reason); - wrapBusinessSender(sender); - return candidate; - } - - function findReceiverForTrack(track) { - if (!track) return null; - return Array.from(window.__aiNativeCopilotReceivers || []) - .find((receiver) => receiver.track === track) || null; - } - - function rememberReceiver(receiver, reason) { - if (!receiver?.track || isInternalPc(receiver.transport)) return; - window.__aiNativeCopilotReceivers.add(receiver); - const track = receiver.track; - if (!isUsableTrack(track)) return; - - if (track.kind === 'audio') { - const candidate = candidateFor(track, audioCandidates); - candidate.receiver = receiver; - candidate.reasons.add(reason); - } else if (track.kind === 'video') { - const candidate = candidateFor(track, videoCandidates); - candidate.receiver = receiver; - candidate.reasons.add(reason); - chooseRemoteVideoTrack(reason); - } - } - - function setVuWidth(id, level) { - const el = document.getElementById(id); - if (!el) return; - el.style.width = `${Math.min(100, level * 2500)}%`; - } - - function renderVu() { - inputVuLevel *= 0.88; - aiOutVuLevel *= 0.88; - setVuWidth('native-copilot-input-vu', inputVuLevel); - setVuWidth('native-copilot-ai-vu', aiOutVuLevel); - requestAnimationFrame(renderVu); - } - - function levelFromStats(stat, memory) { - let level = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0; - if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') { - const prev = memory.get(stat.id); - memory.set(stat.id, { - energy: stat.totalAudioEnergy, - duration: stat.totalSamplesDuration, - }); - if (prev) { - const energyDelta = stat.totalAudioEnergy - prev.energy; - const durationDelta = stat.totalSamplesDuration - prev.duration; - if (energyDelta > 0 && durationDelta > 0) { - level = Math.max(level, Math.sqrt(energyDelta / durationDelta)); - } - } - } - return level; - } - - async function refreshVuFromStats() { - try { - if (!(await checkLockedUserTracks('stats-before'))) return; - await refreshAudioCandidateStats(); - chooseRemoteAudioTrack('stats'); - - await refreshBusinessSenderStats(); - chooseBusinessAudioSender('stats'); - await checkLockedUserTracks('stats-after'); - } catch (error) { - // Stats APIs are best-effort across browsers. - } - } - - async function refreshBusinessSenderStats() { - for (const candidate of businessSenderCandidates.values()) { - const sender = candidate.sender; - if (!sender?.getStats) continue; - candidate.track = sender.track || candidate.track; - - const report = await sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; - - const key = `${candidate.id}:${stat.id}`; - const prev = outboundStatsMemory.get(key); - const bytes = typeof stat.bytesSent === 'number' ? stat.bytesSent : 0; - const packets = typeof stat.packetsSent === 'number' ? stat.packetsSent : 0; - candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - candidate.bytesSent = bytes; - candidate.packetsSent = packets; - outboundStatsMemory.set(key, { bytes, packets }); - - if (candidate.sender === businessAudioSender) { - businessOutboundStats.bytesDelta = candidate.bytesDelta; - businessOutboundStats.packetsDelta = candidate.packetsDelta; - businessOutboundStats.bytesSent = candidate.bytesSent; - businessOutboundStats.packetsSent = candidate.packetsSent; - aiOutVuLevel = Math.max( - aiOutVuLevel, - levelFromStats(stat, senderStatsMemory), - outputVuLevelFromStats() - ); - } - }); - } - } - - async function refreshAudioCandidateStats() { - for (const candidate of audioCandidates.values()) { - if (!isUsableTrack(candidate.track)) continue; - const receiver = candidate.receiver || findReceiverForTrack(candidate.track); - if (!receiver?.getStats) continue; - candidate.receiver = receiver; - - const report = await receiver.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; - - const prev = inboundStatsMemory.get(stat.id); - const bytes = typeof stat.bytesReceived === 'number' ? stat.bytesReceived : 0; - const packets = typeof stat.packetsReceived === 'number' ? stat.packetsReceived : 0; - const bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - const packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - inboundStatsMemory.set(stat.id, { bytes, packets }); - - candidate.bytesReceived = bytes; - candidate.packetsReceived = packets; - candidate.bytesDelta = bytesDelta; - candidate.packetsDelta = packetsDelta; - candidate.level = levelFromStats(stat, receiverStatsMemory); - if (bytesDelta > 0 || packetsDelta > 0 || candidate.level > 0) { - candidate.lastPacketAt = Date.now(); - } - - if (candidate.track === remoteUserAudioTrack) { - inputVuLevel = Math.max(inputVuLevel, inputVuLevelFromCandidate(candidate)); - } - }); - } - } - - async function outboundAudioBytes(sender) { - if (!sender?.getStats) return null; - try { - let total = 0; - const report = await sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type === 'outbound-rtp' && kind === 'audio' && typeof stat.bytesSent === 'number') { - total += stat.bytesSent; - } - }); - return total; - } catch { - return null; - } - } - - function aiReadyForMedia() { - if (!aiPc || !aiPc.remoteDescription) return false; - return ( - aiPc.connectionState === 'connected' - || aiPc.iceConnectionState === 'connected' - || aiPc.iceConnectionState === 'completed' - ); - } - - function aiStateLabel() { - if (!aiPc) return 'pc=null'; - return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; - } - - function waitForAiMediaReady(timeoutMs = 10000) { - if (aiReadyForMedia()) return Promise.resolve(true); - return new Promise((resolve) => { - const startedAt = Date.now(); - const timer = setInterval(() => { - if (aiReadyForMedia()) { - clearInterval(timer); - resolve(true); - } else if (Date.now() - startedAt >= timeoutMs) { - clearInterval(timer); - resolve(false); - } - }, 250); - }); - } - - function scheduleAiInputHealthCheck(track) { - if (aiInputHealthTimer) clearTimeout(aiInputHealthTimer); - aiInputHealthTimer = setTimeout(async () => { - if (!aiAudioSender || aiAudioSender.track !== track) return; - log(`checking native AI input after media connects: ${aiStateLabel()}`); - const ready = await waitForAiMediaReady(10000); - if (!ready || !aiAudioSender || aiAudioSender.track !== track) { - log(`native AI input could not be verified: ${aiStateLabel()}`); - return; - } - - const before = await outboundAudioBytes(aiAudioSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - if (!aiAudioSender || aiAudioSender.track !== track) return; - const after = await outboundAudioBytes(aiAudioSender); - log(`native AI input packet check: before=${before}, after=${after}, ${aiStateLabel()}`, true); - }, 500); - } - - function describeTrack(track) { - if (!track) return 'none'; - return `${track.id}${track.muted ? ' muted' : ''}`; - } - - function elementLayoutInfo(element) { - if (!element?.getBoundingClientRect || !document.documentElement.contains(element)) return null; - const rect = element.getBoundingClientRect(); - const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 1; - const centerY = rect.top + rect.height / 2; - const visibleWidth = Math.max(0, Math.min(rect.right, window.innerWidth || document.documentElement.clientWidth || rect.right) - Math.max(rect.left, 0)); - const visibleHeight = Math.max(0, Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0)); - return { - tag: element.tagName, - top: Math.round(rect.top), - left: Math.round(rect.left), - width: Math.round(rect.width), - height: Math.round(rect.height), - centerY: Math.round(centerY), - upperHalf: centerY <= viewportHeight * 0.55, - visibleArea: Math.round(visibleWidth * visibleHeight), - paused: Boolean(element.paused), - muted: Boolean(element.muted), - volume: typeof element.volume === 'number' ? element.volume : null, - }; - } - - function candidateLayoutScore(candidate) { - let score = 0; - for (const element of candidate?.elements || []) { - const info = elementLayoutInfo(element); - if (!info) continue; - if (info.upperHalf) score += 9000; - else score -= 2500; - if (info.visibleArea > 0) score += Math.min(2500, info.visibleArea / 80); - if (info.tag === 'VIDEO') score += 1200; - if (info.tag === 'AUDIO') score += 300; - if (!info.paused) score += 700; - if (!info.muted && (info.volume ?? 1) > 0) score += 500; - } - return score; - } - - function hasUpperHalfElement(candidate) { - return Array.from(candidate?.elements || []).some((element) => elementLayoutInfo(element)?.upperHalf); - } - - function rootVueStore() { - return document.getElementById('app')?.__vue__?.$store || null; - } - - function streamKeyIndex(key) { - const match = String(key || '').match(/^(?:speaker|custom|layout|patrol|rollCall|speechExcit|demo|focus)_(\d+)_/); - return match ? Number(match[1]) : null; - } - - function audioCandidateBusinessIndex(candidate) { - for (const element of candidate?.elements || []) { - const id = element?.id || ''; - const match = id.match(/^audio[-_](\d+)$/); - if (match) return Number(match[1]); - } - - const trackId = candidate?.track?.id || ''; - const streamId = Array.from(candidate?.elements || []) - .map((element) => element?.srcObject?.id) - .find(Boolean) || trackId; - const streamMatch = String(streamId).match(/audio-(\d+)-a/); - return streamMatch ? Number(streamMatch[1]) : null; - } - - function mediaStoreItems(name) { - const store = rootVueStore(); - const value = store?.getters?.[name]; - return Array.isArray(value) ? value : []; - } - - function upperVideoBusinessIndex() { - const upperVideos = Array.from(document.querySelectorAll('video')) - .map((element) => ({ element, info: elementLayoutInfo(element) })) - .filter(({ element, info }) => info?.upperHalf && info.visibleArea > 0 && element.srcObject instanceof MediaStream) - .sort((a, b) => b.info.visibleArea - a.info.visibleArea); - - const remoteStreams = mediaStoreItems('remoteStreams'); - const allStreams = mediaStoreItems('allStream').concat(mediaStoreItems('subMembers')); - for (const { element } of upperVideos) { - const stream = element.srcObject; - const remote = remoteStreams.find((item) => item?.stream === stream || item?.stream?.id === stream.id); - const streamInfo = remote && allStreams.find((item) => item?.mid === remote.mid); - const index = streamKeyIndex(streamInfo?.key || remote?.key || remote?.key2); - if (index) return index; - } - - return null; - } - - function audioCandidateHasCurrentActivity(candidate) { - return Boolean(candidate && ( - candidate.level >= 0.0015 - || candidate.bytesDelta > 0 - || candidate.packetsDelta > 0 - )); - } - - function audioCandidateHasRecentActivity(candidate) { - return Boolean( - audioCandidateHasCurrentActivity(candidate) - || (candidate && Date.now() - candidate.lastPacketAt < 3000) - ); - } - - function lockedTrackUsable(track, candidates) { - const candidate = candidates.get(track?.id); - return Boolean(candidate && isUsableTrack(candidate.track) && !candidate.track.muted); - } - - async function handleLockedUserTrackLost(kind, reason) { - const message = `${kind === 'audio' ? '远端音频' : '远端视频'}轨道失效（${reason}），已断开 AI`; - appendChatLine('选轨', message, true); - log(message); - if (kind === 'audio') { - remoteUserAudioLocked = false; - remoteUserAudioTrack = null; - } else { - remoteUserVideoLocked = false; - remoteUserVideoTrack = null; - } - if (isAiTakeover) { - isAiTakeover = false; - updateModeButton(); - await restoreBusinessAudio(); - disconnectAi(); - } - } - - async function checkLockedUserTracks(reason) { - if (remoteUserAudioLocked && !lockedTrackUsable(remoteUserAudioTrack, audioCandidates)) { - await handleLockedUserTrackLost('audio', reason); - return false; - } - if (remoteUserVideoLocked && !lockedTrackUsable(remoteUserVideoTrack, videoCandidates)) { - await handleLockedUserTrackLost('video', reason); - return false; - } - return true; - } - - function scoreAudioCandidate(candidate) { - if (!candidate || !isUsableTrack(candidate.track)) return -1; - const hasRecentPackets = Date.now() - candidate.lastPacketAt < 3000; - const hasAudibleLevel = candidate.level >= 0.0015; - const hasCurrentActivity = audioCandidateHasCurrentActivity(candidate); - const layoutScore = candidateLayoutScore(candidate); - const audioIndex = audioCandidateBusinessIndex(candidate); - const targetAudioIndex = upperVideoBusinessIndex(); - let score = 0; - if (candidate.track.muted) score -= 5000; - else score += 800; - score += hasCurrentActivity ? layoutScore : Math.min(1200, layoutScore * 0.1); - if (targetAudioIndex && audioIndex === targetAudioIndex && hasCurrentActivity) score += 8000; - else if (targetAudioIndex && audioIndex && audioIndex !== targetAudioIndex) score -= 2500; - else if (!targetAudioIndex && audioIndex === 1 && hasCurrentActivity) score += 2500; - if (candidate.receiver) score += 600; - if (candidate.bytesReceived > 0) score += 100; - if (candidate.bytesDelta > 0) score += 500 + Math.min(900, candidate.bytesDelta * 2); - if (candidate.packetsDelta > 0) score += 300 + Math.min(500, candidate.packetsDelta * 8); - if (hasRecentPackets) score += 400; - if (hasAudibleLevel) score += 5000; - score += Math.min(12000, candidate.level * 900000); - if (!hasAudibleLevel && hasRecentPackets) score -= 1800; - if (!hasCurrentActivity && layoutScore > 0) score -= 1200; - if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 80; - return score; - } - - function scoreVideoCandidate(candidate) { - if (!candidate || !isUsableTrack(candidate.track)) return -1; - let score = 0; - if (candidate.track.muted) score -= 1000; - else score += 500; - if (candidate.receiver) score += 600; - score += candidateLayoutScore(candidate); - for (const element of candidate.elements || []) { - const info = elementLayoutInfo(element); - if (!info) continue; - if (info.tag === 'VIDEO') score += 2500; - if (info.visibleArea > 0) score += Math.min(4000, info.visibleArea / 50); - } - if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 100; - return score; - } - - function inputVuLevelFromCandidate(candidate) { - if (!candidate) return 0; - return Math.max( - candidate.level, - Math.min(0.04, candidate.bytesDelta / 16000), - Math.min(0.02, candidate.packetsDelta / 800) - ); - } - - function updateSelectedInputLabel() { - const label = document.getElementById('native-copilot-selected-input'); - if (!label) return; - const candidate = audioCandidates.get(remoteUserAudioTrack?.id); - if (!candidate) { - label.textContent = '等待远端音轨'; - return; - } - const layout = Array.from(candidate.elements || []).map(elementLayoutInfo).find(Boolean); - const where = layout ? (layout.upperHalf ? 'upper' : 'lower') : 'no-layout'; - label.textContent = `${remoteUserAudioTrack.id.slice(0, 8)} ${where} level=${candidate.level.toFixed(4)} score=${Math.round(candidate.score || scoreAudioCandidate(candidate))}`; - } - - function outputVuLevelFromStats() { - return Math.max( - Math.min(0.12, businessOutboundStats.bytesDelta / 12000), - Math.min(0.04, businessOutboundStats.packetsDelta / 500) - ); - } - - function isKnownRemoteAudioTrack(track) { - return Boolean(track && ( - track === remoteUserAudioTrack - || audioCandidates.has(track.id) - )); - } - - function isAiOutputTrack(track) { - return Boolean(track && track === aiResponseAudioTrack); - } - - function isUplinkMixerTrack(track) { - return Boolean(track && uplinkStableTrack && track === uplinkStableTrack); - } - - function createAudioContext() { - const AudioContextCtor = window.AudioContext || window.webkitAudioContext; - if (!AudioContextCtor) return null; - return new AudioContextCtor(); - } - - async function ensureUplinkMixer() { - if (!uplinkAudioContext) { - uplinkAudioContext = createAudioContext(); - if (!uplinkAudioContext) { - appendChatLine('音频', '当前浏览器不支持 AudioContext，无法启用稳定上行轨', true); - return false; - } - uplinkDestination = uplinkAudioContext.createMediaStreamDestination(); - uplinkStableTrack = uplinkDestination.stream.getAudioTracks()[0] || null; - manualGain = uplinkAudioContext.createGain(); - aiGain = uplinkAudioContext.createGain(); - manualGain.gain.value = 1; - aiGain.gain.value = 0; - manualGain.connect(uplinkDestination); - aiGain.connect(uplinkDestination); - log(`created stable uplink mixer track ${describeTrack(uplinkStableTrack)}`, true); - } - if (uplinkAudioContext.state === 'suspended') { - await uplinkAudioContext.resume().catch(() => {}); - } - return Boolean(uplinkStableTrack && manualGain && aiGain); - } - - function connectTrackToGain(track, gainNode, currentSource, currentTrack) { - if (!uplinkAudioContext || !gainNode || !isLiveTrack(track)) { - return { source: currentSource, track: currentTrack }; - } - if (currentTrack === track && currentSource) { - return { source: currentSource, track: currentTrack }; - } - try { - currentSource?.disconnect(); - } catch {} - const stream = new MediaStream([track]); - const source = uplinkAudioContext.createMediaStreamSource(stream); - source.connect(gainNode); - return { source, track }; - } - - async function ensureStableBusinessUplink() { - chooseBusinessAudioSender('stable-uplink'); - if (!businessAudioSender) return false; - if (!businessOriginalAudioTrack || isUplinkMixerTrack(businessOriginalAudioTrack)) { - const currentTrack = businessAudioSender.track; - if (currentTrack && !isUplinkMixerTrack(currentTrack)) { - businessOriginalAudioTrack = currentTrack; - } - } - if (!isLiveTrack(businessOriginalAudioTrack)) { - appendChatLine('音频', '没有可用的原业务上行音轨，无法创建稳定上行轨', true); - return false; - } - if (!(await ensureUplinkMixer())) return false; - - const manual = connectTrackToGain( - businessOriginalAudioTrack, - manualGain, - manualSource, - manualSourceTrack, - ); - manualSource = manual.source; - manualSourceTrack = manual.track; - - if (businessAudioSender.track !== uplinkStableTrack) { - await businessAudioSender.replaceTrack(uplinkStableTrack); - log(`business call now uses stable uplink mixer track (${uplinkStableTrack.id})`, true); - } - aiTrackAttachedToBusiness = true; - return true; - } - - function setStableUplinkMode(mode) { - if (!manualGain || !aiGain) return; - const now = uplinkAudioContext?.currentTime || 0; - const manualValue = mode === 'ai' ? 0 : 1; - const aiValue = mode === 'ai' ? 1 : 0; - try { - manualGain.gain.cancelScheduledValues(now); - aiGain.gain.cancelScheduledValues(now); - manualGain.gain.setTargetAtTime(manualValue, now, 0.015); - aiGain.gain.setTargetAtTime(aiValue, now, 0.015); - } catch { - manualGain.gain.value = manualValue; - aiGain.gain.value = aiValue; - } - log(`stable uplink mode=${mode}`); - } - - function scoreBusinessSenderCandidate(candidate) { - if (!candidate?.sender) return -1; - const track = candidate.sender.track || candidate.track; - let score = 0; - - if (track?.kind === 'audio') score += 500; - else if (!track) score -= 1000; - - if (track?.readyState === 'live') score += 700; - if (track && !track.muted) score += 300; - if (candidate.sources.has('pc-addTrack')) score += 5000; - if (candidate.sources.has('pc-addTransceiver')) score -= 1000; - if (track && localCaptureTrackIds.has(track.id)) score += 2500; - if (candidate.originalTrack && localCaptureTrackIds.has(candidate.originalTrack.id)) score += 2500; - if (isKnownRemoteAudioTrack(track)) score -= 9000; - if (candidate.originalTrack && isKnownRemoteAudioTrack(candidate.originalTrack)) score -= 9000; - if (isAiOutputTrack(track)) score += 4500; - if (isUplinkMixerTrack(track)) score += 4500; - if (candidate.bytesSent > 0) score += 600; - if (candidate.bytesDelta > 0) score += 1600 + Math.min(1800, candidate.bytesDelta * 4); - if (candidate.packetsDelta > 0) score += 800 + Math.min(900, candidate.packetsDelta * 10); - - return score; - } - - function chooseBusinessAudioSender(reason) { - let best = null; - let bestScore = -1; - for (const candidate of businessSenderCandidates.values()) { - candidate.score = scoreBusinessSenderCandidate(candidate); - if (candidate.score > bestScore) { - best = candidate; - bestScore = candidate.score; - } - } - - if (!best || bestScore < 0) return false; - if (businessAudioSender === best.sender) return true; - - businessAudioSender = best.sender; - businessOriginalAudioTrack = best.originalTrack || best.sender.track || businessOriginalAudioTrack; - log(`selected business audio sender (${reason}): ${best.id}, track=${describeTrack(best.sender.track)}, score=${Math.round(bestScore)}, sources=${Array.from(best.sources).join('+')}`, true); - attachAiOutputToBusiness(); - return true; - } - - function chooseRemoteAudioTrack(reason) { - const lockedAudio = audioCandidates.get(remoteUserAudioTrack?.id); - if (remoteUserAudioLocked && lockedAudio && isUsableTrack(lockedAudio.track) && !lockedAudio.track.muted) { - if (audioCandidateHasRecentActivity(lockedAudio)) { - updateSelectedInputLabel(); - return true; - } - remoteUserAudioLocked = false; - } - if (remoteUserAudioLocked) return false; - - let best = null; - let bestScore = -1; - for (const candidate of audioCandidates.values()) { - candidate.score = scoreAudioCandidate(candidate); - if (candidate.score > bestScore) { - best = candidate; - bestScore = candidate.score; - } - } - - if (!best || bestScore < 0) return false; - if (remoteUserAudioTrack === best.track) { - updateSelectedInputLabel(); - return true; - } - - const current = audioCandidates.get(remoteUserAudioTrack?.id); - const currentScore = scoreAudioCandidate(current); - const currentHealthy = current && isUsableTrack(current.track) && !current.track.muted - && audioCandidateHasRecentActivity(current); - if (currentHealthy && bestScore < currentScore + 1800) { - updateSelectedInputLabel(); - return true; - } - - remoteUserAudioTrack = best.track; - remoteUserAudioLocked = hasUpperHalfElement(best) && audioCandidateHasRecentActivity(best); - log(`${remoteUserAudioLocked ? 'locked' : 'selected'} remote audio track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, level=${best.level.toFixed(4)}, bytesDelta=${best.bytesDelta}, packetsDelta=${best.packetsDelta}`, true); - updateSelectedInputLabel(); - attachRemoteInputToAi(); - return true; - } - - function chooseRemoteVideoTrack(reason) { - const lockedVideo = videoCandidates.get(remoteUserVideoTrack?.id); - if (remoteUserVideoLocked && lockedVideo && isUsableTrack(lockedVideo.track) && !lockedVideo.track.muted) { - return true; - } - if (remoteUserVideoLocked) return false; - - let best = null; - let bestScore = -1; - for (const candidate of videoCandidates.values()) { - candidate.score = scoreVideoCandidate(candidate); - if (candidate.score > bestScore) { - best = candidate; - bestScore = candidate.score; - } - } - - if (!best || bestScore < 0) return false; - if (remoteUserVideoTrack === best.track) return true; - - const current = videoCandidates.get(remoteUserVideoTrack?.id); - const currentScore = scoreVideoCandidate(current); - if (hasUpperHalfElement(current) && bestScore < currentScore + 1800) return true; - - remoteUserVideoTrack = best.track; - remoteUserVideoLocked = hasUpperHalfElement(best); - log(`${remoteUserVideoLocked ? 'locked' : 'selected'} remote video track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, layout=${Math.round(candidateLayoutScore(best))}`, true); - attachRemoteVideoToAi(); - return true; - } - - function ensureUserTracksBeforeConnect() { - chooseRemoteAudioTrack('pre-connect'); - chooseRemoteVideoTrack('pre-connect'); - - if (shouldSendAudioToAi() && (!remoteUserAudioLocked || !lockedTrackUsable(remoteUserAudioTrack, audioCandidates))) { - appendChatLine('选轨', '未锁定上半部用户音频轨，暂不连接 AI', true); - return false; - } - if (shouldSendVideoToAi() && (!remoteUserVideoLocked || !lockedTrackUsable(remoteUserVideoTrack, videoCandidates))) { - appendChatLine('选轨', '未锁定上半部用户视频轨，暂不连接 AI', true); - return false; - } - return true; - } - - function rememberRemoteTrack(track, reason, element = null) { - if (!isUsableTrack(track)) return; - - if (track.kind === 'audio') { - const candidate = candidateFor(track, audioCandidates); - candidate.receiver = candidate.receiver || findReceiverForTrack(track); - candidate.reasons.add(reason); - if (element) candidate.elements.add(element); - if (!candidate.logged) { - candidate.logged = true; - log(`captured remote audio candidate (${reason}): ${describeTrack(track)}`, true); - } - chooseRemoteAudioTrack(reason); - return; - } - - if (track.kind === 'video') { - const candidate = candidateFor(track, videoCandidates); - candidate.receiver = candidate.receiver || findReceiverForTrack(track); - candidate.reasons.add(reason); - if (element) candidate.elements.add(element); - if (!candidate.logged) { - candidate.logged = true; - log(`captured remote video candidate (${reason}): ${describeTrack(track)}`, true); - } - chooseRemoteVideoTrack(reason); - } - } - - function rememberRemoteStream(stream, reason, element = null) { - if (!stream || !(stream instanceof MediaStream)) return; - if (stream.__aiNativeCopilotInternal) return; - - const liveTracks = stream.getTracks().filter(isUsableTrack); - if (liveTracks.length === 0) return; - - remoteStreams.add(stream); - for (const track of liveTracks) rememberRemoteTrack(track, reason, element); - chooseRemoteVideoTrack(reason); - } - - function rememberBusinessAudioSender(sender, track, reason) { - const candidate = businessSenderCandidateFor(sender, track, reason); - if (!candidate) return; - if (!candidate.logged) { - candidate.logged = true; - log(`captured business audio sender candidate (${reason}): ${candidate.id}, track=${describeTrack(candidate.track)}`, true); - } - chooseBusinessAudioSender(reason); - } - - function wrapBusinessSender(sender) { - if (!sender || sender.__aiNativeCopilotWrapped) return; - const originalReplaceTrack = sender.replaceTrack?.bind(sender); - if (!originalReplaceTrack) return; - - sender.replaceTrack = async function (track) { - const result = await originalReplaceTrack(track); - const candidate = businessSenderCandidateFor(sender, track, 'replaceTrack'); - if (candidate) { - candidate.track = track || sender.track || candidate.track; - } - if (!isAiTakeover && track?.kind === 'audio' && track !== aiResponseAudioTrack && !isUplinkMixerTrack(track)) { - if (candidate) candidate.originalTrack = track; - chooseBusinessAudioSender('replaceTrack'); - } - return result; - }; - sender.__aiNativeCopilotWrapped = true; - } - - async function attachRemoteInputToAi() { - if (!shouldSendAudioToAi()) return false; - if (!aiAudioSender || !isUsableTrack(remoteUserAudioTrack)) return false; - if (aiAudioSender.track === remoteUserAudioTrack) return true; - - try { - await aiAudioSender.replaceTrack(remoteUserAudioTrack); - log(`AI input uses native remote audio track (${remoteUserAudioTrack.id})`, true); - scheduleAiInputHealthCheck(remoteUserAudioTrack); - return true; - } catch (error) { - log(`failed to attach native remote audio to AI: ${error.message || error}`); - return false; - } - } - - async function attachRemoteVideoToAi() { - if (!shouldSendVideoToAi()) return false; - if (!aiVideoSender || !isUsableTrack(remoteUserVideoTrack)) return false; - if (aiVideoSender.track === remoteUserVideoTrack) return true; - - try { - await aiVideoSender.replaceTrack(remoteUserVideoTrack); - log('AI video uses native remote video track', true); - return true; - } catch (error) { - log(`failed to attach native remote video to AI: ${error.message || error}`); - return false; - } - } - - async function attachAiOutputToBusiness() { - chooseBusinessAudioSender('before-ai-output'); - if (!isAiTakeover || !businessAudioSender || !isLiveTrack(aiResponseAudioTrack)) { - log(`AI output waiting: takeover=${isAiTakeover}, businessSender=${Boolean(businessAudioSender)}, aiTrack=${describeTrack(aiResponseAudioTrack)}`); - return false; - } - if (businessAudioSender.track === aiResponseAudioTrack) return true; - - try { - await businessAudioSender.replaceTrack(aiResponseAudioTrack); - aiTrackAttachedToBusiness = true; - log(`business call now uses direct AI response audio track (${aiResponseAudioTrack.id})`, true); - scheduleAiOutputHealthCheck(aiResponseAudioTrack); - return true; - } catch (error) { - aiTrackAttachedToBusiness = false; - log(`failed to attach AI audio to business call: ${error.message || error}`); - return false; - } - } - - function scheduleAiOutputHealthCheck(track) { - if (aiOutputHealthTimer) clearTimeout(aiOutputHealthTimer); - aiOutputHealthTimer = setTimeout(async () => { - if (!businessAudioSender || businessAudioSender.track !== track) { - log(`AI output check skipped: senderTrack=${describeTrack(businessAudioSender?.track)}, aiTrack=${describeTrack(track)}`); - return; - } - - const before = await outboundAudioBytes(businessAudioSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - if (!businessAudioSender || businessAudioSender.track !== track) return; - const after = await outboundAudioBytes(businessAudioSender); - log(`AI output packet check: before=${before}, after=${after}, senderTrack=${describeTrack(businessAudioSender.track)}`, true); - }, 800); - } - - async function restoreBusinessAudio() { - if (!businessAudioSender || !aiTrackAttachedToBusiness) return; - - try { - await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null); - log('business call restored original audio track', true); - } catch (error) { - log(`failed to restore business audio track: ${error.message || error}`); - } finally { - aiTrackAttachedToBusiness = false; - } - } - - function wrapTrackListener(listener) { - if (!listener || listener.__aiNativeCopilotWrapped) return listener; - - const wrapped = function (event) { - if (!isInternalPc(this) && event?.track) { - rememberReceiver(event.receiver, 'pc-track'); - rememberRemoteTrack(event.track, 'pc-track'); - for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); - } - - if (typeof listener === 'function') return listener.apply(this, arguments); - if (listener?.handleEvent) return listener.handleEvent(event); - return undefined; - }; - - wrapped.__aiNativeCopilotWrapped = true; - return wrapped; - } - - RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { - if (type === 'track') { - return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); - } - return originalAddEventListener.call(this, type, listener, options); - }; - - try { - const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); - Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { - configurable: true, - enumerable: true, - get() { - return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeCopilotOnTrack || null; - }, - set(listener) { - const wrapped = wrapTrackListener(listener); - this.__aiNativeCopilotOnTrack = wrapped; - if (descriptor?.set) descriptor.set.call(this, wrapped); - else this.addEventListener('track', wrapped); - }, - }); - } catch (error) { - console.warn('[AI Native Copilot] failed to wrap ontrack', error); - } - - RTCPeerConnection.prototype.addTrack = function (track, ...streams) { - const sender = originalAddTrack.call(this, track, ...streams); - if (!isInternalPc(this) && track?.kind === 'audio') { - rememberBusinessAudioSender(sender, track, 'pc-addTrack'); - } - return sender; - }; - - if (originalAddTransceiver) { - RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { - const transceiver = originalAddTransceiver.call(this, trackOrKind, init); - if (!isInternalPc(this)) { - const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; - if (kind === 'audio') { - const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; - rememberBusinessAudioSender(transceiver.sender, track, 'pc-addTransceiver'); - } - } - return transceiver; - }; - } - - RTCPeerConnection.prototype.setRemoteDescription = function () { - const result = originalSetRemoteDescription.apply(this, arguments); - if (!isInternalPc(this)) { - Promise.resolve(result).then(() => { - try { - for (const receiver of this.getReceivers?.() || []) { - rememberReceiver(receiver, 'receiver-after-srd'); - rememberRemoteTrack(receiver.track, 'receiver-after-srd'); - } - for (const sender of this.getSenders?.() || []) { - rememberBusinessAudioSender(sender, sender.track, 'sender-after-srd'); - } - for (const stream of this.getRemoteStreams?.() || []) { - rememberRemoteStream(stream, 'remote-stream-after-srd'); - } - } catch (error) { - console.warn('[AI Native Copilot] failed to scan receivers', error); - } - }); - } - return result; - }; - - if (mediaSrcObjectDescriptor?.set) { - Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { - configurable: true, - enumerable: mediaSrcObjectDescriptor.enumerable, - get() { - return mediaSrcObjectDescriptor.get.call(this); - }, - set(stream) { - if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiNativeCopilotElement) { - rememberRemoteStream(stream, 'media-srcObject', this); - } - return mediaSrcObjectDescriptor.set.call(this, stream); - }, - }); - } - - if (originalGetUserMedia) { - navigator.mediaDevices.getUserMedia = async function (constraints) { - const stream = await originalGetUserMedia(constraints); - for (const track of stream.getAudioTracks()) { - localCaptureTrackIds.add(track.id); - } - chooseBusinessAudioSender('getUserMedia'); - renderPanelSoon(); - return stream; - }; - } - - setInterval(async () => { - if (!(await checkLockedUserTracks('scan'))) return; - for (const element of document.querySelectorAll('audio, video')) { - if (element.__aiNativeCopilotElement) continue; - rememberRemoteStream(element.srcObject, 'media-scan', element); - } - - for (const stream of Array.from(remoteStreams)) { - rememberRemoteStream(stream, 'stream-rescan'); - } - await checkLockedUserTracks('scan-after'); - }, 1000); - - window.__aiNativeCopilotDump = function () { - const audio = Array.from(audioCandidates.values()).map((candidate) => ({ - id: candidate.track.id, - label: candidate.track.label, - selected: candidate.track === remoteUserAudioTrack, - enabled: candidate.track.enabled, - muted: candidate.track.muted, - readyState: candidate.track.readyState, - hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)), - bytesReceived: candidate.bytesReceived, - packetsReceived: candidate.packetsReceived, - bytesDelta: candidate.bytesDelta, - packetsDelta: candidate.packetsDelta, - level: candidate.level, - audible: candidate.level >= 0.0015, - layoutScore: Math.round(candidateLayoutScore(candidate)), - elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean), - vuLevel: inputVuLevelFromCandidate(candidate), - score: Math.round(scoreAudioCandidate(candidate)), - reasons: Array.from(candidate.reasons), - })); - const video = Array.from(videoCandidates.values()).map((candidate) => ({ - id: candidate.track.id, - label: candidate.track.label, - selected: candidate.track === remoteUserVideoTrack, - muted: candidate.track.muted, - readyState: candidate.track.readyState, - hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)), - layoutScore: Math.round(candidateLayoutScore(candidate)), - score: Math.round(scoreVideoCandidate(candidate)), - elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean), - reasons: Array.from(candidate.reasons), - })); - const businessSenders = Array.from(businessSenderCandidates.values()).map((candidate) => ({ - id: candidate.id, - selected: candidate.sender === businessAudioSender, - track: describeTrack(candidate.sender.track || candidate.track), - originalTrack: describeTrack(candidate.originalTrack), - isRemoteTrack: isKnownRemoteAudioTrack(candidate.sender.track || candidate.track), - isLocalCaptureTrack: Boolean(candidate.sender.track && localCaptureTrackIds.has(candidate.sender.track.id)), - bytesSent: candidate.bytesSent, - packetsSent: candidate.packetsSent, - bytesDelta: candidate.bytesDelta, - packetsDelta: candidate.packetsDelta, - score: Math.round(scoreBusinessSenderCandidate(candidate)), - sources: Array.from(candidate.sources), - })); - return { - outputMode: 'direct-ai-track', - selectedAudio: describeTrack(remoteUserAudioTrack), - selectedVideo: describeTrack(remoteUserVideoTrack), - audioLocked: remoteUserAudioLocked, - videoLocked: remoteUserVideoLocked, - aiInputTrack: describeTrack(aiAudioSender?.track), - aiOutputTrack: describeTrack(aiResponseAudioTrack), - businessSenderTrack: describeTrack(businessAudioSender?.track), - aiTrackAttachedToBusiness, - videoRecording: videoRecorder?.state === 'recording' ? { - track: describeTrack(videoRecordingTrack), - seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1), - chunks: videoRecordingChunks.length, - } : null, - businessOutboundStats: { ...businessOutboundStats }, - aiState: aiStateLabel(), - audio, - video, - businessSenders, - }; - }; - - async function connectAi(signalingUrl, assistantId) { - if (!ensureUserTracksBeforeConnect()) { - setStatus('disconnected'); - return false; - } - disconnectAi(); - - const pcId = generatePcId(); - aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); - aiPc.__aiNativeCopilotInternal = true; - aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); - aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); - - aiDc = aiPc.createDataChannel('chat'); - aiDc.onopen = () => { - setStatus('connected'); - log('AI data channel open', true); - aiDc.send(JSON.stringify({ type: 'client-ready' })); - }; - aiDc.onclose = () => setStatus('disconnected'); - aiDc.onmessage = (event) => { - try { - const msg = JSON.parse(event.data); - const content = msg.delta || msg.content; - if (msg.type === 'transcript') { - logUserTranscript(content); - } else if (msg.type === 'assistant-text-start') { - startAssistantLine(); - } else if (msg.type === 'assistant-text-delta') { - appendAssistantDelta(content); - } else if (msg.type === 'assistant-text-end') { - finishAssistantLine(); - } else { - log(`data channel message: ${msg.type}`); - } - } catch { - appendChatLine('DATA', event.data); - } - }; - - const sendAudio = shouldSendAudioToAi(); - const audioTransceiver = sendAudio && isUsableTrack(remoteUserAudioTrack) - ? aiPc.addTransceiver(remoteUserAudioTrack, { direction: 'sendrecv' }) - : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); - aiAudioSender = audioTransceiver.sender; - log(!sendAudio - ? 'AI audio uplink disabled by panel setting' - : isUsableTrack(remoteUserAudioTrack) - ? 'AI input starts with native remote audio track' - : 'AI input waiting for native remote audio track', true); - - const sendVideo = shouldSendVideoToAi(); - if (sendVideo) { - const videoTransceiver = isUsableTrack(remoteUserVideoTrack) - ? aiPc.addTransceiver(remoteUserVideoTrack, { direction: 'sendonly' }) - : aiPc.addTransceiver('video', { direction: 'sendonly' }); - aiVideoSender = videoTransceiver.sender; - if (isUsableTrack(remoteUserVideoTrack)) log('AI video starts with native remote video track', true); - } else { - aiVideoSender = null; - log('AI video uplink disabled by panel setting'); - } - if (sendAudio && isUsableTrack(remoteUserAudioTrack)) scheduleAiInputHealthCheck(remoteUserAudioTrack); - - aiPc.ontrack = (event) => { - if (event.track.kind !== 'audio') return; - ignoredTrackIds.add(event.track.id); - aiResponseAudioTrack = event.track; - log('received native AI response audio track', true); - attachAiOutputToBusiness(); - }; - - aiPc.onicecandidate = (event) => { - if (aiWs?.readyState !== WebSocket.OPEN) return; - aiWs.send(JSON.stringify({ - type: 'ice-candidate', - payload: { - pc_id: pcId, - candidate: event.candidate ? { - candidate: event.candidate.candidate, - sdpMid: event.candidate.sdpMid, - sdpMLineIndex: event.candidate.sdpMLineIndex, - } : null, - }, - })); - }; - - setStatus('connecting'); - aiWs = new WebSocket(signalingUrl); - aiWs.onopen = async () => { - const offer = await aiPc.createOffer(); - await aiPc.setLocalDescription(offer); - aiWs.send(JSON.stringify({ - type: 'offer', - payload: { - pc_id: pcId, - sdp: aiPc.localDescription.sdp, - type: aiPc.localDescription.type, - assistant_id: assistantId, - vision_enabled: sendVideo, - }, - })); - log('sent AI WebRTC offer', true); - }; - aiWs.onmessage = async (event) => { - try { - const msg = JSON.parse(event.data); - if (msg.type === 'answer') { - await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); - log(`AI WebRTC answer applied; ${aiStateLabel()}`, true); - } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { - await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); - } else if (msg.type === 'error') { - throw new Error(msg.payload?.message || 'server error'); - } - } catch (error) { - log(`signaling error: ${error.message || error}`); - } - }; - aiWs.onerror = () => { - setStatus('failed'); - log('AI signaling websocket error'); - }; - aiWs.onclose = () => setStatus('disconnected'); - - attachRemoteInputToAi(); - if (sendVideo) attachRemoteVideoToAi(); - return true; - } - - function disconnectAi() { - if (aiDc) { - try { aiDc.close(); } catch {} - aiDc = null; - } - if (aiPc) { - try { aiPc.close(); } catch {} - aiPc = null; - } - if (aiWs) { - try { aiWs.close(); } catch {} - aiWs = null; - } - aiAudioSender = null; - aiVideoSender = null; - aiResponseAudioTrack = null; - if (inputRecorder?.state === 'recording') stopInputRecording(); - if (videoRecorder?.state === 'recording') stopVideoRecording(); - setStatus('disconnected'); - } - - function setStatus(status) { - const el = document.getElementById('native-copilot-status'); - if (!el) return; - const labels = { - disconnected: '未连接', - connecting: '连接中', - connected: '已连接', - failed: '连接失败', - }; - el.textContent = labels[status] || status; - } - - function updateModeButton() { - const button = document.getElementById('native-copilot-toggle'); - if (!button) return; - if (isAiTakeover) { - button.textContent = '当前：AI 接管｜点击切回人工'; - button.style.background = '#8b5cf6'; - } else { - button.textContent = '当前：人工接管｜点击切换 AI'; - button.style.background = '#10b981'; - } - } - - function renderPanel() { - if (!document.body || document.getElementById('native-copilot-panel')) return; - - const panel = document.createElement('div'); - panel.id = 'native-copilot-panel'; - panel.style.cssText = [ - 'position:fixed', - 'top:20px', - 'left:20px', - 'width:360px', - 'z-index:999999', - 'box-sizing:border-box', - 'padding:16px', - 'border-radius:10px', - 'background:rgba(17,24,39,.96)', - 'color:white', - 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', - 'box-shadow:0 12px 32px rgba(0,0,0,.35)', - ].join(';'); - - panel.innerHTML = ` -
- AI 原生选轨 - 未连接 -
-
- 音频输入：布局优先自动选轨
- 视频输入：远端视频轨 -
- - - - -
- - -
-
-
- 等待远端音轨原生音轨 -
-
-
-
-
-
-
- AI 输出业务上行 -
-
-
-
-
- -
- - -
-
- 等待交互... -
- `; - - document.body.appendChild(panel); - - const toggle = document.getElementById('native-copilot-toggle'); - const recordButton = document.getElementById('native-copilot-record'); - const videoRecordButton = document.getElementById('native-copilot-record-video'); - const urlInput = document.getElementById('native-copilot-url'); - const assistantInput = document.getElementById('native-copilot-assistant'); - const sendAudioInput = document.getElementById('native-copilot-send-audio'); - const sendVideoInput = document.getElementById('native-copilot-send-video'); - - sendAudioInput.checked = shouldSendAudioToAi(); - sendVideoInput.checked = shouldSendVideoToAi(); - sendAudioInput.addEventListener('change', () => { - writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked); - log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`); - }); - sendVideoInput.addEventListener('change', () => { - writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked); - log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`); - }); - recordButton.addEventListener('click', toggleInputRecording); - videoRecordButton.addEventListener('click', toggleVideoRecording); - updateRecordButton(); - updateVideoRecordButton(); - updateSelectedInputLabel(); - updateModeButton(); - - toggle.addEventListener('click', async () => { - if (!isAiTakeover && !assistantInput.value.trim()) { - alert('请先填写助手 ID。'); - return; - } - - if (!isAiTakeover && !ensureUserTracksBeforeConnect()) { - updateModeButton(); - return; - } - - isAiTakeover = !isAiTakeover; - updateModeButton(); - if (isAiTakeover) { - const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim()); - if (connected) await attachAiOutputToBusiness(); - else { - isAiTakeover = false; - updateModeButton(); - } - } else { - await restoreBusinessAudio(); - disconnectAi(); - } - }); - } - - function renderPanelSoon() { - try { renderPanel(); } catch (error) { console.warn('[AI Native Copilot] panel error', error); } - } - - if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true }); - } else { - renderPanelSoon(); - } - requestAnimationFrame(renderVu); - setInterval(refreshVuFromStats, 250); - setTimeout(renderPanelSoon, 1500); - setTimeout(renderPanelSoon, 3000); -})(); diff --git a/scripts/copilot-native.js b/scripts/copilot-native.js deleted file mode 100644 index c340435..0000000 --- a/scripts/copilot-native.js +++ /dev/null @@ -1,1542 +0,0 @@ -// ==UserScript== -// @name 12345 AI Native Track Copilot -// @namespace http://tampermonkey.net/ -// @version 1.0 -// @description Native-track only bridge: no WebAudio washing, no captureStream, no MediaStreamDestination. -// @author You -// @match *://yhy-yw.22sj.cn:*/* -// @grant none -// @run-at document-start -// ==/UserScript== - -(function () { - 'use strict'; - - const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; - const AUDIO_UPLINK_STORAGE_KEY = 'native-copilot-send-audio'; - const VIDEO_UPLINK_STORAGE_KEY = 'native-copilot-send-video'; - - const originalGetUserMedia = navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices); - const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; - const originalAddTrack = RTCPeerConnection.prototype.addTrack; - const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; - const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; - const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); - - const remoteStreams = new Set(); - const ignoredTrackIds = new Set(); - const audioCandidates = new Map(); - const videoCandidates = new Map(); - const businessSenderCandidates = new Map(); - const localCaptureTrackIds = new Set(); - window.__aiNativeCopilotReceivers = window.__aiNativeCopilotReceivers || new Set(); - - let aiPc = null; - let aiWs = null; - let aiDc = null; - let aiAudioSender = null; - let aiVideoSender = null; - - let businessAudioSender = null; - let businessOriginalAudioTrack = null; - - let remoteUserAudioTrack = null; - let remoteUserVideoTrack = null; - let aiResponseAudioTrack = null; - let remoteUserAudioLocked = false; - let remoteUserVideoLocked = false; - - let isAiTakeover = false; - let aiTrackAttachedToBusiness = false; - let inputVuLevel = 0; - let aiOutVuLevel = 0; - let aiInputHealthTimer = null; - let aiOutputHealthTimer = null; - let assistantLine = null; - let assistantBuffer = ''; - let inputRecorder = null; - let inputRecordingChunks = []; - let inputRecordingTrack = null; - let inputRecordingStartedAt = 0; - let videoRecorder = null; - let videoRecordingChunks = []; - let videoRecordingTrack = null; - let videoRecordingStartedAt = 0; - - const receiverStatsMemory = new Map(); - const senderStatsMemory = new Map(); - const inboundStatsMemory = new Map(); - const outboundStatsMemory = new Map(); - const businessOutboundStats = { - bytesSent: 0, - packetsSent: 0, - bytesDelta: 0, - packetsDelta: 0, - }; - - function log(message) { - console.log(`[AI Native Copilot] ${message}`); - } - - function readBoolSetting(key, fallback) { - try { - const value = localStorage.getItem(key); - if (value === null) return fallback; - return value === 'true'; - } catch { - return fallback; - } - } - - function writeBoolSetting(key, value) { - try { - localStorage.setItem(key, value ? 'true' : 'false'); - } catch { - // Keep the live checkbox state even if storage is unavailable. - } - } - - function shouldSendAudioToAi() { - return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); - } - - function shouldSendVideoToAi() { - return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); - } - - function appendChatLine(role, text, important = false) { - const box = document.getElementById('native-copilot-log'); - if (!box) return; - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - - const row = document.createElement('div'); - row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; - row.style.marginBottom = '4px'; - if (important) row.style.color = '#fcd34d'; - box.appendChild(row); - while (box.children.length > 120) box.removeChild(box.firstChild); - box.scrollTop = box.scrollHeight; - return row; - } - - function startAssistantLine() { - assistantBuffer = ''; - assistantLine = null; - } - - function appendAssistantDelta(delta) { - if (!delta) return; - const box = document.getElementById('native-copilot-log'); - if (!box) { - console.log(`[AI Native Copilot] AI: ${delta}`); - return; - } - if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; - if (!assistantLine) assistantLine = appendChatLine('AI', ''); - assistantBuffer += delta; - assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; - box.scrollTop = box.scrollHeight; - } - - function finishAssistantLine() { - assistantLine = null; - assistantBuffer = ''; - } - - function logUserTranscript(text) { - if (!text?.trim()) return; - finishAssistantLine(); - appendChatLine('用户', text.trim(), true); - } - - function recorderMimeType() { - if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; - if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; - return ''; - } - - function recordingFileName(track) { - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; - return `native-ai-input-${trackPart}-${stamp}.webm`; - } - - function videoRecordingFileName(track) { - const stamp = new Date().toISOString().replace(/[:.]/g, '-'); - const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; - return `native-ai-video-${trackPart}-${stamp}.webm`; - } - - function updateRecordButton() { - const button = document.getElementById('native-copilot-record'); - if (!button) return; - const recording = inputRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; - button.style.background = recording ? '#dc2626' : '#2563eb'; - } - - function videoRecorderMimeType() { - if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8'; - if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm'; - return ''; - } - - function updateVideoRecordButton() { - const button = document.getElementById('native-copilot-record-video'); - if (!button) return; - const recording = videoRecorder?.state === 'recording'; - button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频'; - button.style.background = recording ? '#dc2626' : '#475569'; - } - - async function startInputRecording() { - if (inputRecorder?.state === 'recording') return; - chooseRemoteAudioTrack('record'); - if (!isUsableTrack(remoteUserAudioTrack)) { - appendChatLine('录音', '没有可录制的 AI 输入音轨', true); - return; - } - if (!window.MediaRecorder) { - appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); - return; - } - - inputRecordingChunks = []; - inputRecordingTrack = remoteUserAudioTrack; - inputRecordingStartedAt = Date.now(); - const stream = new MediaStream([inputRecordingTrack]); - const mimeType = recorderMimeType(); - inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); - inputRecorder.ondataavailable = (event) => { - if (event.data?.size > 0) inputRecordingChunks.push(event.data); - }; - inputRecorder.onstop = () => { - const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - link.download = recordingFileName(inputRecordingTrack); - document.body.appendChild(link); - link.click(); - link.remove(); - setTimeout(() => URL.revokeObjectURL(url), 30000); - - const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); - appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); - inputRecorder = null; - inputRecordingChunks = []; - inputRecordingTrack = null; - updateRecordButton(); - }; - inputRecorder.start(250); - appendChatLine('录音', `开始录制 AI 输入：${describeTrack(inputRecordingTrack)}`, true); - updateRecordButton(); - } - - function stopInputRecording() { - if (inputRecorder?.state === 'recording') inputRecorder.stop(); - } - - function toggleInputRecording() { - if (inputRecorder?.state === 'recording') stopInputRecording(); - else startInputRecording(); - } - - async function startVideoRecording() { - if (videoRecorder?.state === 'recording') return; - chooseRemoteVideoTrack('record-video'); - if (!remoteUserVideoLocked || !isUsableTrack(remoteUserVideoTrack)) { - appendChatLine('录制视频', '没有可录制的上半部用户视频轨', true); - return; - } - if (!window.MediaRecorder) { - appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true); - return; - } - - videoRecordingChunks = []; - videoRecordingTrack = remoteUserVideoTrack; - videoRecordingStartedAt = Date.now(); - const stream = new MediaStream([videoRecordingTrack]); - const mimeType = videoRecorderMimeType(); - videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); - videoRecorder.ondataavailable = (event) => { - if (event.data?.size > 0) videoRecordingChunks.push(event.data); - }; - videoRecorder.onstop = () => { - const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' }); - const url = URL.createObjectURL(blob); - const link = document.createElement('a'); - link.href = url; - link.download = videoRecordingFileName(videoRecordingTrack); - document.body.appendChild(link); - link.click(); - link.remove(); - setTimeout(() => URL.revokeObjectURL(url), 30000); - - const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1); - appendChatLine('录制视频', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true); - videoRecorder = null; - videoRecordingChunks = []; - videoRecordingTrack = null; - updateVideoRecordButton(); - }; - videoRecorder.start(250); - appendChatLine('录制视频', `开始录制 AI 视频：${describeTrack(videoRecordingTrack)}`, true); - updateVideoRecordButton(); - } - - function stopVideoRecording() { - if (videoRecorder?.state === 'recording') videoRecorder.stop(); - } - - function toggleVideoRecording() { - if (videoRecorder?.state === 'recording') stopVideoRecording(); - else startVideoRecording(); - } - - function generatePcId() { - const bytes = new Uint8Array(16); - crypto.getRandomValues(bytes); - return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); - } - - function isInternalPc(pc) { - return Boolean(pc?.__aiNativeCopilotInternal); - } - - function isUsableTrack(track) { - return track && track.readyState === 'live' && !ignoredTrackIds.has(track.id); - } - - function isLiveTrack(track) { - return track && track.readyState === 'live'; - } - - function candidateFor(track, candidates) { - let candidate = candidates.get(track.id); - if (!candidate) { - candidate = { - track, - receiver: null, - reasons: new Set(), - elements: new Set(), - firstSeenAt: Date.now(), - lastSeenAt: 0, - lastPacketAt: 0, - bytesReceived: 0, - packetsReceived: 0, - bytesDelta: 0, - packetsDelta: 0, - level: 0, - score: 0, - logged: false, - }; - candidates.set(track.id, candidate); - } - candidate.track = track; - candidate.lastSeenAt = Date.now(); - return candidate; - } - - function businessSenderCandidateFor(sender, track, reason) { - if (!sender) return null; - const senderTrack = track || sender.track; - if (senderTrack && senderTrack.kind !== 'audio') return null; - - const id = sender.__aiNativeCopilotBusinessSenderId || `business-sender-${businessSenderCandidates.size + 1}`; - sender.__aiNativeCopilotBusinessSenderId = id; - - let candidate = businessSenderCandidates.get(id); - if (!candidate) { - candidate = { - id, - sender, - track: senderTrack || null, - originalTrack: senderTrack || null, - sources: new Set(), - bytesSent: 0, - packetsSent: 0, - bytesDelta: 0, - packetsDelta: 0, - score: 0, - logged: false, - }; - businessSenderCandidates.set(id, candidate); - } - - candidate.sender = sender; - candidate.track = senderTrack || sender.track || candidate.track; - if (!isAiTakeover && senderTrack && senderTrack !== aiResponseAudioTrack) { - candidate.originalTrack = senderTrack; - } - candidate.sources.add(reason); - wrapBusinessSender(sender); - return candidate; - } - - function findReceiverForTrack(track) { - if (!track) return null; - return Array.from(window.__aiNativeCopilotReceivers || []) - .find((receiver) => receiver.track === track) || null; - } - - function rememberReceiver(receiver, reason) { - if (!receiver?.track || isInternalPc(receiver.transport)) return; - window.__aiNativeCopilotReceivers.add(receiver); - const track = receiver.track; - if (!isUsableTrack(track)) return; - - if (track.kind === 'audio') { - const candidate = candidateFor(track, audioCandidates); - candidate.receiver = receiver; - candidate.reasons.add(reason); - } else if (track.kind === 'video') { - const candidate = candidateFor(track, videoCandidates); - candidate.receiver = receiver; - candidate.reasons.add(reason); - chooseRemoteVideoTrack(reason); - } - } - - function setVuWidth(id, level) { - const el = document.getElementById(id); - if (!el) return; - el.style.width = `${Math.min(100, level * 2500)}%`; - } - - function renderVu() { - inputVuLevel *= 0.88; - aiOutVuLevel *= 0.88; - setVuWidth('native-copilot-input-vu', inputVuLevel); - setVuWidth('native-copilot-ai-vu', aiOutVuLevel); - requestAnimationFrame(renderVu); - } - - function levelFromStats(stat, memory) { - let level = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0; - if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') { - const prev = memory.get(stat.id); - memory.set(stat.id, { - energy: stat.totalAudioEnergy, - duration: stat.totalSamplesDuration, - }); - if (prev) { - const energyDelta = stat.totalAudioEnergy - prev.energy; - const durationDelta = stat.totalSamplesDuration - prev.duration; - if (energyDelta > 0 && durationDelta > 0) { - level = Math.max(level, Math.sqrt(energyDelta / durationDelta)); - } - } - } - return level; - } - - async function refreshVuFromStats() { - try { - if (!(await checkLockedUserTracks('stats-before'))) return; - await refreshAudioCandidateStats(); - chooseRemoteAudioTrack('stats'); - - await refreshBusinessSenderStats(); - chooseBusinessAudioSender('stats'); - await checkLockedUserTracks('stats-after'); - } catch (error) { - // Stats APIs are best-effort across browsers. - } - } - - async function refreshBusinessSenderStats() { - for (const candidate of businessSenderCandidates.values()) { - const sender = candidate.sender; - if (!sender?.getStats) continue; - candidate.track = sender.track || candidate.track; - - const report = await sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; - - const key = `${candidate.id}:${stat.id}`; - const prev = outboundStatsMemory.get(key); - const bytes = typeof stat.bytesSent === 'number' ? stat.bytesSent : 0; - const packets = typeof stat.packetsSent === 'number' ? stat.packetsSent : 0; - candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - candidate.bytesSent = bytes; - candidate.packetsSent = packets; - outboundStatsMemory.set(key, { bytes, packets }); - - if (candidate.sender === businessAudioSender) { - businessOutboundStats.bytesDelta = candidate.bytesDelta; - businessOutboundStats.packetsDelta = candidate.packetsDelta; - businessOutboundStats.bytesSent = candidate.bytesSent; - businessOutboundStats.packetsSent = candidate.packetsSent; - aiOutVuLevel = Math.max( - aiOutVuLevel, - levelFromStats(stat, senderStatsMemory), - outputVuLevelFromStats() - ); - } - }); - } - } - - async function refreshAudioCandidateStats() { - for (const candidate of audioCandidates.values()) { - if (!isUsableTrack(candidate.track)) continue; - const receiver = candidate.receiver || findReceiverForTrack(candidate.track); - if (!receiver?.getStats) continue; - candidate.receiver = receiver; - - const report = await receiver.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; - - const prev = inboundStatsMemory.get(stat.id); - const bytes = typeof stat.bytesReceived === 'number' ? stat.bytesReceived : 0; - const packets = typeof stat.packetsReceived === 'number' ? stat.packetsReceived : 0; - const bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; - const packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; - inboundStatsMemory.set(stat.id, { bytes, packets }); - - candidate.bytesReceived = bytes; - candidate.packetsReceived = packets; - candidate.bytesDelta = bytesDelta; - candidate.packetsDelta = packetsDelta; - candidate.level = levelFromStats(stat, receiverStatsMemory); - if (bytesDelta > 0 || packetsDelta > 0 || candidate.level > 0) { - candidate.lastPacketAt = Date.now(); - } - - if (candidate.track === remoteUserAudioTrack) { - inputVuLevel = Math.max(inputVuLevel, inputVuLevelFromCandidate(candidate)); - } - }); - } - } - - async function outboundAudioBytes(sender) { - if (!sender?.getStats) return null; - try { - let total = 0; - const report = await sender.getStats(); - report.forEach((stat) => { - const kind = stat.kind || stat.mediaType; - if (stat.type === 'outbound-rtp' && kind === 'audio' && typeof stat.bytesSent === 'number') { - total += stat.bytesSent; - } - }); - return total; - } catch { - return null; - } - } - - function aiReadyForMedia() { - if (!aiPc || !aiPc.remoteDescription) return false; - return ( - aiPc.connectionState === 'connected' - || aiPc.iceConnectionState === 'connected' - || aiPc.iceConnectionState === 'completed' - ); - } - - function aiStateLabel() { - if (!aiPc) return 'pc=null'; - return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; - } - - function waitForAiMediaReady(timeoutMs = 10000) { - if (aiReadyForMedia()) return Promise.resolve(true); - return new Promise((resolve) => { - const startedAt = Date.now(); - const timer = setInterval(() => { - if (aiReadyForMedia()) { - clearInterval(timer); - resolve(true); - } else if (Date.now() - startedAt >= timeoutMs) { - clearInterval(timer); - resolve(false); - } - }, 250); - }); - } - - function scheduleAiInputHealthCheck(track) { - if (aiInputHealthTimer) clearTimeout(aiInputHealthTimer); - aiInputHealthTimer = setTimeout(async () => { - if (!aiAudioSender || aiAudioSender.track !== track) return; - log(`checking native AI input after media connects: ${aiStateLabel()}`); - const ready = await waitForAiMediaReady(10000); - if (!ready || !aiAudioSender || aiAudioSender.track !== track) { - log(`native AI input could not be verified: ${aiStateLabel()}`); - return; - } - - const before = await outboundAudioBytes(aiAudioSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - if (!aiAudioSender || aiAudioSender.track !== track) return; - const after = await outboundAudioBytes(aiAudioSender); - log(`native AI input packet check: before=${before}, after=${after}, ${aiStateLabel()}`, true); - }, 500); - } - - function describeTrack(track) { - if (!track) return 'none'; - return `${track.id}${track.muted ? ' muted' : ''}`; - } - - function elementLayoutInfo(element) { - if (!element?.getBoundingClientRect || !document.documentElement.contains(element)) return null; - const rect = element.getBoundingClientRect(); - const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 1; - const centerY = rect.top + rect.height / 2; - const visibleWidth = Math.max(0, Math.min(rect.right, window.innerWidth || document.documentElement.clientWidth || rect.right) - Math.max(rect.left, 0)); - const visibleHeight = Math.max(0, Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0)); - return { - tag: element.tagName, - top: Math.round(rect.top), - left: Math.round(rect.left), - width: Math.round(rect.width), - height: Math.round(rect.height), - centerY: Math.round(centerY), - upperHalf: centerY <= viewportHeight * 0.55, - visibleArea: Math.round(visibleWidth * visibleHeight), - paused: Boolean(element.paused), - muted: Boolean(element.muted), - volume: typeof element.volume === 'number' ? element.volume : null, - }; - } - - function candidateLayoutScore(candidate) { - let score = 0; - for (const element of candidate?.elements || []) { - const info = elementLayoutInfo(element); - if (!info) continue; - if (info.upperHalf) score += 9000; - else score -= 2500; - if (info.visibleArea > 0) score += Math.min(2500, info.visibleArea / 80); - if (info.tag === 'VIDEO') score += 1200; - if (info.tag === 'AUDIO') score += 300; - if (!info.paused) score += 700; - if (!info.muted && (info.volume ?? 1) > 0) score += 500; - } - return score; - } - - function hasUpperHalfElement(candidate) { - return Array.from(candidate?.elements || []).some((element) => elementLayoutInfo(element)?.upperHalf); - } - - function lockedTrackUsable(track, candidates) { - const candidate = candidates.get(track?.id); - return Boolean(candidate && isUsableTrack(candidate.track) && !candidate.track.muted); - } - - async function handleLockedUserTrackLost(kind, reason) { - const message = `${kind === 'audio' ? '远端音频' : '远端视频'}轨道失效（${reason}），已断开 AI`; - appendChatLine('选轨', message, true); - log(message); - if (kind === 'audio') { - remoteUserAudioLocked = false; - remoteUserAudioTrack = null; - } else { - remoteUserVideoLocked = false; - remoteUserVideoTrack = null; - } - if (isAiTakeover) { - isAiTakeover = false; - updateModeButton(); - await restoreBusinessAudio(); - disconnectAi(); - } - } - - async function checkLockedUserTracks(reason) { - if (remoteUserAudioLocked && !lockedTrackUsable(remoteUserAudioTrack, audioCandidates)) { - await handleLockedUserTrackLost('audio', reason); - return false; - } - if (remoteUserVideoLocked && !lockedTrackUsable(remoteUserVideoTrack, videoCandidates)) { - await handleLockedUserTrackLost('video', reason); - return false; - } - return true; - } - - function scoreAudioCandidate(candidate) { - if (!candidate || !isUsableTrack(candidate.track)) return -1; - const hasRecentPackets = Date.now() - candidate.lastPacketAt < 3000; - const hasAudibleLevel = candidate.level >= 0.0015; - let score = 0; - if (candidate.track.muted) score -= 5000; - else score += 800; - score += candidateLayoutScore(candidate); - if (candidate.receiver) score += 600; - if (candidate.bytesReceived > 0) score += 100; - if (candidate.bytesDelta > 0) score += 500 + Math.min(900, candidate.bytesDelta * 2); - if (candidate.packetsDelta > 0) score += 300 + Math.min(500, candidate.packetsDelta * 8); - if (hasRecentPackets) score += 400; - if (hasAudibleLevel) score += 5000; - score += Math.min(12000, candidate.level * 900000); - if (!hasAudibleLevel && hasRecentPackets) score -= 1800; - if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 80; - return score; - } - - function scoreVideoCandidate(candidate) { - if (!candidate || !isUsableTrack(candidate.track)) return -1; - let score = 0; - if (candidate.track.muted) score -= 1000; - else score += 500; - if (candidate.receiver) score += 600; - score += candidateLayoutScore(candidate); - for (const element of candidate.elements || []) { - const info = elementLayoutInfo(element); - if (!info) continue; - if (info.tag === 'VIDEO') score += 2500; - if (info.visibleArea > 0) score += Math.min(4000, info.visibleArea / 50); - } - if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 100; - return score; - } - - function inputVuLevelFromCandidate(candidate) { - if (!candidate) return 0; - return Math.max( - candidate.level, - Math.min(0.04, candidate.bytesDelta / 16000), - Math.min(0.02, candidate.packetsDelta / 800) - ); - } - - function updateSelectedInputLabel() { - const label = document.getElementById('native-copilot-selected-input'); - if (!label) return; - const candidate = audioCandidates.get(remoteUserAudioTrack?.id); - if (!candidate) { - label.textContent = '等待远端音轨'; - return; - } - const layout = Array.from(candidate.elements || []).map(elementLayoutInfo).find(Boolean); - const where = layout ? (layout.upperHalf ? 'upper' : 'lower') : 'no-layout'; - label.textContent = `${remoteUserAudioTrack.id.slice(0, 8)} ${where} level=${candidate.level.toFixed(4)} score=${Math.round(candidate.score || scoreAudioCandidate(candidate))}`; - } - - function outputVuLevelFromStats() { - return Math.max( - Math.min(0.12, businessOutboundStats.bytesDelta / 12000), - Math.min(0.04, businessOutboundStats.packetsDelta / 500) - ); - } - - function isKnownRemoteAudioTrack(track) { - return Boolean(track && ( - track === remoteUserAudioTrack - || audioCandidates.has(track.id) - )); - } - - function isAiOutputTrack(track) { - return Boolean(track && track === aiResponseAudioTrack); - } - - function scoreBusinessSenderCandidate(candidate) { - if (!candidate?.sender) return -1; - const track = candidate.sender.track || candidate.track; - let score = 0; - - if (track?.kind === 'audio') score += 500; - else if (!track) score -= 1000; - - if (track?.readyState === 'live') score += 700; - if (track && !track.muted) score += 300; - if (candidate.sources.has('pc-addTrack')) score += 5000; - if (candidate.sources.has('pc-addTransceiver')) score -= 1000; - if (track && localCaptureTrackIds.has(track.id)) score += 2500; - if (candidate.originalTrack && localCaptureTrackIds.has(candidate.originalTrack.id)) score += 2500; - if (isKnownRemoteAudioTrack(track)) score -= 9000; - if (candidate.originalTrack && isKnownRemoteAudioTrack(candidate.originalTrack)) score -= 9000; - if (isAiOutputTrack(track)) score += 4500; - if (candidate.bytesSent > 0) score += 600; - if (candidate.bytesDelta > 0) score += 1600 + Math.min(1800, candidate.bytesDelta * 4); - if (candidate.packetsDelta > 0) score += 800 + Math.min(900, candidate.packetsDelta * 10); - - return score; - } - - function chooseBusinessAudioSender(reason) { - let best = null; - let bestScore = -1; - for (const candidate of businessSenderCandidates.values()) { - candidate.score = scoreBusinessSenderCandidate(candidate); - if (candidate.score > bestScore) { - best = candidate; - bestScore = candidate.score; - } - } - - if (!best || bestScore < 0) return false; - if (businessAudioSender === best.sender) return true; - - businessAudioSender = best.sender; - businessOriginalAudioTrack = best.originalTrack || best.sender.track || businessOriginalAudioTrack; - log(`selected business audio sender (${reason}): ${best.id}, track=${describeTrack(best.sender.track)}, score=${Math.round(bestScore)}, sources=${Array.from(best.sources).join('+')}`, true); - attachAiOutputToBusiness(); - return true; - } - - function chooseRemoteAudioTrack(reason) { - const lockedAudio = audioCandidates.get(remoteUserAudioTrack?.id); - if (remoteUserAudioLocked && lockedAudio && isUsableTrack(lockedAudio.track) && !lockedAudio.track.muted) { - updateSelectedInputLabel(); - return true; - } - if (remoteUserAudioLocked) return false; - - let best = null; - let bestScore = -1; - for (const candidate of audioCandidates.values()) { - candidate.score = scoreAudioCandidate(candidate); - if (candidate.score > bestScore) { - best = candidate; - bestScore = candidate.score; - } - } - - if (!best || bestScore < 0) return false; - if (remoteUserAudioTrack === best.track) { - updateSelectedInputLabel(); - return true; - } - - const current = audioCandidates.get(remoteUserAudioTrack?.id); - const currentScore = scoreAudioCandidate(current); - const currentHealthy = current && isUsableTrack(current.track) && !current.track.muted - && (current.level >= 0.0015 || hasUpperHalfElement(current)) - && Date.now() - current.lastPacketAt < 3000; - if (currentHealthy && bestScore < currentScore + 1800) { - updateSelectedInputLabel(); - return true; - } - - remoteUserAudioTrack = best.track; - remoteUserAudioLocked = hasUpperHalfElement(best); - log(`${remoteUserAudioLocked ? 'locked' : 'selected'} remote audio track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, level=${best.level.toFixed(4)}, bytesDelta=${best.bytesDelta}, packetsDelta=${best.packetsDelta}`, true); - updateSelectedInputLabel(); - attachRemoteInputToAi(); - return true; - } - - function chooseRemoteVideoTrack(reason) { - const lockedVideo = videoCandidates.get(remoteUserVideoTrack?.id); - if (remoteUserVideoLocked && lockedVideo && isUsableTrack(lockedVideo.track) && !lockedVideo.track.muted) { - return true; - } - if (remoteUserVideoLocked) return false; - - let best = null; - let bestScore = -1; - for (const candidate of videoCandidates.values()) { - candidate.score = scoreVideoCandidate(candidate); - if (candidate.score > bestScore) { - best = candidate; - bestScore = candidate.score; - } - } - - if (!best || bestScore < 0) return false; - if (remoteUserVideoTrack === best.track) return true; - - const current = videoCandidates.get(remoteUserVideoTrack?.id); - const currentScore = scoreVideoCandidate(current); - if (hasUpperHalfElement(current) && bestScore < currentScore + 1800) return true; - - remoteUserVideoTrack = best.track; - remoteUserVideoLocked = hasUpperHalfElement(best); - log(`${remoteUserVideoLocked ? 'locked' : 'selected'} remote video track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, layout=${Math.round(candidateLayoutScore(best))}`, true); - attachRemoteVideoToAi(); - return true; - } - - function ensureUserTracksBeforeConnect() { - chooseRemoteAudioTrack('pre-connect'); - chooseRemoteVideoTrack('pre-connect'); - - if (shouldSendAudioToAi() && (!remoteUserAudioLocked || !lockedTrackUsable(remoteUserAudioTrack, audioCandidates))) { - appendChatLine('选轨', '未锁定上半部用户音频轨，暂不连接 AI', true); - return false; - } - if (shouldSendVideoToAi() && (!remoteUserVideoLocked || !lockedTrackUsable(remoteUserVideoTrack, videoCandidates))) { - appendChatLine('选轨', '未锁定上半部用户视频轨，暂不连接 AI', true); - return false; - } - return true; - } - - function rememberRemoteTrack(track, reason, element = null) { - if (!isUsableTrack(track)) return; - - if (track.kind === 'audio') { - const candidate = candidateFor(track, audioCandidates); - candidate.receiver = candidate.receiver || findReceiverForTrack(track); - candidate.reasons.add(reason); - if (element) candidate.elements.add(element); - if (!candidate.logged) { - candidate.logged = true; - log(`captured remote audio candidate (${reason}): ${describeTrack(track)}`, true); - } - chooseRemoteAudioTrack(reason); - return; - } - - if (track.kind === 'video') { - const candidate = candidateFor(track, videoCandidates); - candidate.receiver = candidate.receiver || findReceiverForTrack(track); - candidate.reasons.add(reason); - if (element) candidate.elements.add(element); - if (!candidate.logged) { - candidate.logged = true; - log(`captured remote video candidate (${reason}): ${describeTrack(track)}`, true); - } - chooseRemoteVideoTrack(reason); - } - } - - function rememberRemoteStream(stream, reason, element = null) { - if (!stream || !(stream instanceof MediaStream)) return; - if (stream.__aiNativeCopilotInternal) return; - - const liveTracks = stream.getTracks().filter(isUsableTrack); - if (liveTracks.length === 0) return; - - remoteStreams.add(stream); - for (const track of liveTracks) rememberRemoteTrack(track, reason, element); - chooseRemoteVideoTrack(reason); - } - - function rememberBusinessAudioSender(sender, track, reason) { - const candidate = businessSenderCandidateFor(sender, track, reason); - if (!candidate) return; - if (!candidate.logged) { - candidate.logged = true; - log(`captured business audio sender candidate (${reason}): ${candidate.id}, track=${describeTrack(candidate.track)}`, true); - } - chooseBusinessAudioSender(reason); - } - - function wrapBusinessSender(sender) { - if (!sender || sender.__aiNativeCopilotWrapped) return; - const originalReplaceTrack = sender.replaceTrack?.bind(sender); - if (!originalReplaceTrack) return; - - sender.replaceTrack = async function (track) { - const result = await originalReplaceTrack(track); - const candidate = businessSenderCandidateFor(sender, track, 'replaceTrack'); - if (candidate) { - candidate.track = track || sender.track || candidate.track; - } - if (!isAiTakeover && track?.kind === 'audio' && track !== aiResponseAudioTrack) { - if (candidate) candidate.originalTrack = track; - chooseBusinessAudioSender('replaceTrack'); - } - return result; - }; - sender.__aiNativeCopilotWrapped = true; - } - - async function attachRemoteInputToAi() { - if (!shouldSendAudioToAi()) return false; - if (!aiAudioSender || !isUsableTrack(remoteUserAudioTrack)) return false; - if (aiAudioSender.track === remoteUserAudioTrack) return true; - - try { - await aiAudioSender.replaceTrack(remoteUserAudioTrack); - log(`AI input uses native remote audio track (${remoteUserAudioTrack.id})`, true); - scheduleAiInputHealthCheck(remoteUserAudioTrack); - return true; - } catch (error) { - log(`failed to attach native remote audio to AI: ${error.message || error}`); - return false; - } - } - - async function attachRemoteVideoToAi() { - if (!shouldSendVideoToAi()) return false; - if (!aiVideoSender || !isUsableTrack(remoteUserVideoTrack)) return false; - if (aiVideoSender.track === remoteUserVideoTrack) return true; - - try { - await aiVideoSender.replaceTrack(remoteUserVideoTrack); - log('AI video uses native remote video track', true); - return true; - } catch (error) { - log(`failed to attach native remote video to AI: ${error.message || error}`); - return false; - } - } - - async function attachAiOutputToBusiness() { - chooseBusinessAudioSender('before-ai-output'); - if (!isAiTakeover || !businessAudioSender || !isLiveTrack(aiResponseAudioTrack)) { - log(`AI output waiting: takeover=${isAiTakeover}, businessSender=${Boolean(businessAudioSender)}, aiTrack=${describeTrack(aiResponseAudioTrack)}`); - return false; - } - if (businessAudioSender.track === aiResponseAudioTrack) return true; - - try { - await businessAudioSender.replaceTrack(aiResponseAudioTrack); - aiTrackAttachedToBusiness = true; - log(`business call now uses native AI response audio track (${aiResponseAudioTrack.id})`, true); - scheduleAiOutputHealthCheck(aiResponseAudioTrack); - return true; - } catch (error) { - aiTrackAttachedToBusiness = false; - log(`failed to attach AI audio to business call: ${error.message || error}`); - return false; - } - } - - function scheduleAiOutputHealthCheck(track) { - if (aiOutputHealthTimer) clearTimeout(aiOutputHealthTimer); - aiOutputHealthTimer = setTimeout(async () => { - if (!businessAudioSender || businessAudioSender.track !== track) { - log(`AI output check skipped: senderTrack=${describeTrack(businessAudioSender?.track)}, aiTrack=${describeTrack(track)}`); - return; - } - - const before = await outboundAudioBytes(businessAudioSender); - await new Promise((resolve) => setTimeout(resolve, 2500)); - if (!businessAudioSender || businessAudioSender.track !== track) return; - const after = await outboundAudioBytes(businessAudioSender); - log(`AI output packet check: before=${before}, after=${after}, senderTrack=${describeTrack(businessAudioSender.track)}`, true); - }, 800); - } - - async function restoreBusinessAudio() { - if (!businessAudioSender || !aiTrackAttachedToBusiness) return; - - try { - await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null); - log('business call restored original audio track', true); - } catch (error) { - log(`failed to restore business audio track: ${error.message || error}`); - } finally { - aiTrackAttachedToBusiness = false; - } - } - - function wrapTrackListener(listener) { - if (!listener || listener.__aiNativeCopilotWrapped) return listener; - - const wrapped = function (event) { - if (!isInternalPc(this) && event?.track) { - rememberReceiver(event.receiver, 'pc-track'); - rememberRemoteTrack(event.track, 'pc-track'); - for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); - } - - if (typeof listener === 'function') return listener.apply(this, arguments); - if (listener?.handleEvent) return listener.handleEvent(event); - return undefined; - }; - - wrapped.__aiNativeCopilotWrapped = true; - return wrapped; - } - - RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { - if (type === 'track') { - return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); - } - return originalAddEventListener.call(this, type, listener, options); - }; - - try { - const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); - Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { - configurable: true, - enumerable: true, - get() { - return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeCopilotOnTrack || null; - }, - set(listener) { - const wrapped = wrapTrackListener(listener); - this.__aiNativeCopilotOnTrack = wrapped; - if (descriptor?.set) descriptor.set.call(this, wrapped); - else this.addEventListener('track', wrapped); - }, - }); - } catch (error) { - console.warn('[AI Native Copilot] failed to wrap ontrack', error); - } - - RTCPeerConnection.prototype.addTrack = function (track, ...streams) { - const sender = originalAddTrack.call(this, track, ...streams); - if (!isInternalPc(this) && track?.kind === 'audio') { - rememberBusinessAudioSender(sender, track, 'pc-addTrack'); - } - return sender; - }; - - if (originalAddTransceiver) { - RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { - const transceiver = originalAddTransceiver.call(this, trackOrKind, init); - if (!isInternalPc(this)) { - const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; - if (kind === 'audio') { - const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; - rememberBusinessAudioSender(transceiver.sender, track, 'pc-addTransceiver'); - } - } - return transceiver; - }; - } - - RTCPeerConnection.prototype.setRemoteDescription = function () { - const result = originalSetRemoteDescription.apply(this, arguments); - if (!isInternalPc(this)) { - Promise.resolve(result).then(() => { - try { - for (const receiver of this.getReceivers?.() || []) { - rememberReceiver(receiver, 'receiver-after-srd'); - rememberRemoteTrack(receiver.track, 'receiver-after-srd'); - } - for (const sender of this.getSenders?.() || []) { - rememberBusinessAudioSender(sender, sender.track, 'sender-after-srd'); - } - for (const stream of this.getRemoteStreams?.() || []) { - rememberRemoteStream(stream, 'remote-stream-after-srd'); - } - } catch (error) { - console.warn('[AI Native Copilot] failed to scan receivers', error); - } - }); - } - return result; - }; - - if (mediaSrcObjectDescriptor?.set) { - Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { - configurable: true, - enumerable: mediaSrcObjectDescriptor.enumerable, - get() { - return mediaSrcObjectDescriptor.get.call(this); - }, - set(stream) { - if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiNativeCopilotElement) { - rememberRemoteStream(stream, 'media-srcObject', this); - } - return mediaSrcObjectDescriptor.set.call(this, stream); - }, - }); - } - - if (originalGetUserMedia) { - navigator.mediaDevices.getUserMedia = async function (constraints) { - const stream = await originalGetUserMedia(constraints); - for (const track of stream.getAudioTracks()) { - localCaptureTrackIds.add(track.id); - } - chooseBusinessAudioSender('getUserMedia'); - renderPanelSoon(); - return stream; - }; - } - - setInterval(async () => { - if (!(await checkLockedUserTracks('scan'))) return; - for (const element of document.querySelectorAll('audio, video')) { - if (element.__aiNativeCopilotElement) continue; - rememberRemoteStream(element.srcObject, 'media-scan', element); - } - - for (const stream of Array.from(remoteStreams)) { - rememberRemoteStream(stream, 'stream-rescan'); - } - await checkLockedUserTracks('scan-after'); - }, 1000); - - window.__aiNativeCopilotDump = function () { - const audio = Array.from(audioCandidates.values()).map((candidate) => ({ - id: candidate.track.id, - label: candidate.track.label, - selected: candidate.track === remoteUserAudioTrack, - enabled: candidate.track.enabled, - muted: candidate.track.muted, - readyState: candidate.track.readyState, - hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)), - bytesReceived: candidate.bytesReceived, - packetsReceived: candidate.packetsReceived, - bytesDelta: candidate.bytesDelta, - packetsDelta: candidate.packetsDelta, - level: candidate.level, - audible: candidate.level >= 0.0015, - layoutScore: Math.round(candidateLayoutScore(candidate)), - elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean), - vuLevel: inputVuLevelFromCandidate(candidate), - score: Math.round(scoreAudioCandidate(candidate)), - reasons: Array.from(candidate.reasons), - })); - const video = Array.from(videoCandidates.values()).map((candidate) => ({ - id: candidate.track.id, - label: candidate.track.label, - selected: candidate.track === remoteUserVideoTrack, - muted: candidate.track.muted, - readyState: candidate.track.readyState, - hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)), - layoutScore: Math.round(candidateLayoutScore(candidate)), - score: Math.round(scoreVideoCandidate(candidate)), - elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean), - reasons: Array.from(candidate.reasons), - })); - const businessSenders = Array.from(businessSenderCandidates.values()).map((candidate) => ({ - id: candidate.id, - selected: candidate.sender === businessAudioSender, - track: describeTrack(candidate.sender.track || candidate.track), - originalTrack: describeTrack(candidate.originalTrack), - isRemoteTrack: isKnownRemoteAudioTrack(candidate.sender.track || candidate.track), - isLocalCaptureTrack: Boolean(candidate.sender.track && localCaptureTrackIds.has(candidate.sender.track.id)), - bytesSent: candidate.bytesSent, - packetsSent: candidate.packetsSent, - bytesDelta: candidate.bytesDelta, - packetsDelta: candidate.packetsDelta, - score: Math.round(scoreBusinessSenderCandidate(candidate)), - sources: Array.from(candidate.sources), - })); - return { - selectedAudio: describeTrack(remoteUserAudioTrack), - selectedVideo: describeTrack(remoteUserVideoTrack), - audioLocked: remoteUserAudioLocked, - videoLocked: remoteUserVideoLocked, - aiInputTrack: describeTrack(aiAudioSender?.track), - aiOutputTrack: describeTrack(aiResponseAudioTrack), - businessSenderTrack: describeTrack(businessAudioSender?.track), - aiTrackAttachedToBusiness, - videoRecording: videoRecorder?.state === 'recording' ? { - track: describeTrack(videoRecordingTrack), - seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1), - chunks: videoRecordingChunks.length, - } : null, - businessOutboundStats: { ...businessOutboundStats }, - aiState: aiStateLabel(), - audio, - video, - businessSenders, - }; - }; - - async function connectAi(signalingUrl, assistantId) { - if (!ensureUserTracksBeforeConnect()) { - setStatus('disconnected'); - return false; - } - disconnectAi(); - - const pcId = generatePcId(); - aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); - aiPc.__aiNativeCopilotInternal = true; - aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); - aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); - - aiDc = aiPc.createDataChannel('chat'); - aiDc.onopen = () => { - setStatus('connected'); - log('AI data channel open', true); - aiDc.send(JSON.stringify({ type: 'client-ready' })); - }; - aiDc.onclose = () => setStatus('disconnected'); - aiDc.onmessage = (event) => { - try { - const msg = JSON.parse(event.data); - const content = msg.delta || msg.content; - if (msg.type === 'transcript') { - logUserTranscript(content); - } else if (msg.type === 'assistant-text-start') { - startAssistantLine(); - } else if (msg.type === 'assistant-text-delta') { - appendAssistantDelta(content); - } else if (msg.type === 'assistant-text-end') { - finishAssistantLine(); - } else { - log(`data channel message: ${msg.type}`); - } - } catch { - appendChatLine('DATA', event.data); - } - }; - - const sendAudio = shouldSendAudioToAi(); - const audioTransceiver = sendAudio && isUsableTrack(remoteUserAudioTrack) - ? aiPc.addTransceiver(remoteUserAudioTrack, { direction: 'sendrecv' }) - : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); - aiAudioSender = audioTransceiver.sender; - log(!sendAudio - ? 'AI audio uplink disabled by panel setting' - : isUsableTrack(remoteUserAudioTrack) - ? 'AI input starts with native remote audio track' - : 'AI input waiting for native remote audio track', true); - - const sendVideo = shouldSendVideoToAi(); - if (sendVideo) { - const videoTransceiver = isUsableTrack(remoteUserVideoTrack) - ? aiPc.addTransceiver(remoteUserVideoTrack, { direction: 'sendonly' }) - : aiPc.addTransceiver('video', { direction: 'sendonly' }); - aiVideoSender = videoTransceiver.sender; - if (isUsableTrack(remoteUserVideoTrack)) log('AI video starts with native remote video track', true); - } else { - aiVideoSender = null; - log('AI video uplink disabled by panel setting'); - } - if (sendAudio && isUsableTrack(remoteUserAudioTrack)) scheduleAiInputHealthCheck(remoteUserAudioTrack); - - aiPc.ontrack = (event) => { - if (event.track.kind !== 'audio') return; - ignoredTrackIds.add(event.track.id); - aiResponseAudioTrack = event.track; - log('received native AI response audio track', true); - attachAiOutputToBusiness(); - }; - - aiPc.onicecandidate = (event) => { - if (aiWs?.readyState !== WebSocket.OPEN) return; - aiWs.send(JSON.stringify({ - type: 'ice-candidate', - payload: { - pc_id: pcId, - candidate: event.candidate ? { - candidate: event.candidate.candidate, - sdpMid: event.candidate.sdpMid, - sdpMLineIndex: event.candidate.sdpMLineIndex, - } : null, - }, - })); - }; - - setStatus('connecting'); - aiWs = new WebSocket(signalingUrl); - aiWs.onopen = async () => { - const offer = await aiPc.createOffer(); - await aiPc.setLocalDescription(offer); - aiWs.send(JSON.stringify({ - type: 'offer', - payload: { - pc_id: pcId, - sdp: aiPc.localDescription.sdp, - type: aiPc.localDescription.type, - assistant_id: assistantId, - vision_enabled: sendVideo, - }, - })); - log('sent AI WebRTC offer', true); - }; - aiWs.onmessage = async (event) => { - try { - const msg = JSON.parse(event.data); - if (msg.type === 'answer') { - await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); - log(`AI WebRTC answer applied; ${aiStateLabel()}`, true); - } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { - await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); - } else if (msg.type === 'error') { - throw new Error(msg.payload?.message || 'server error'); - } - } catch (error) { - log(`signaling error: ${error.message || error}`); - } - }; - aiWs.onerror = () => { - setStatus('failed'); - log('AI signaling websocket error'); - }; - aiWs.onclose = () => setStatus('disconnected'); - - attachRemoteInputToAi(); - if (sendVideo) attachRemoteVideoToAi(); - return true; - } - - function disconnectAi() { - if (aiDc) { - try { aiDc.close(); } catch {} - aiDc = null; - } - if (aiPc) { - try { aiPc.close(); } catch {} - aiPc = null; - } - if (aiWs) { - try { aiWs.close(); } catch {} - aiWs = null; - } - aiAudioSender = null; - aiVideoSender = null; - aiResponseAudioTrack = null; - if (inputRecorder?.state === 'recording') stopInputRecording(); - if (videoRecorder?.state === 'recording') stopVideoRecording(); - setStatus('disconnected'); - } - - function setStatus(status) { - const el = document.getElementById('native-copilot-status'); - if (!el) return; - const labels = { - disconnected: '未连接', - connecting: '连接中', - connected: '已连接', - failed: '连接失败', - }; - el.textContent = labels[status] || status; - } - - function updateModeButton() { - const button = document.getElementById('native-copilot-toggle'); - if (!button) return; - if (isAiTakeover) { - button.textContent = '当前：AI 接管｜点击切回人工'; - button.style.background = '#8b5cf6'; - } else { - button.textContent = '当前：人工接管｜点击切换 AI'; - button.style.background = '#10b981'; - } - } - - function renderPanel() { - if (!document.body || document.getElementById('native-copilot-panel')) return; - - const panel = document.createElement('div'); - panel.id = 'native-copilot-panel'; - panel.style.cssText = [ - 'position:fixed', - 'top:20px', - 'left:20px', - 'width:360px', - 'z-index:999999', - 'box-sizing:border-box', - 'padding:16px', - 'border-radius:10px', - 'background:rgba(17,24,39,.96)', - 'color:white', - 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', - 'box-shadow:0 12px 32px rgba(0,0,0,.35)', - ].join(';'); - - panel.innerHTML = ` -
- AI 原生选轨 - 未连接 -
-
- 音频输入：布局优先自动选轨
- 视频输入：远端视频轨 -
- - - - -
- - -
-
-
- 等待远端音轨原生音轨 -
-
-
-
-
-
-
- AI 输出业务上行 -
-
-
-
-
- -
- - -
-
- 等待交互... -
- `; - - document.body.appendChild(panel); - - const toggle = document.getElementById('native-copilot-toggle'); - const recordButton = document.getElementById('native-copilot-record'); - const videoRecordButton = document.getElementById('native-copilot-record-video'); - const urlInput = document.getElementById('native-copilot-url'); - const assistantInput = document.getElementById('native-copilot-assistant'); - const sendAudioInput = document.getElementById('native-copilot-send-audio'); - const sendVideoInput = document.getElementById('native-copilot-send-video'); - - sendAudioInput.checked = shouldSendAudioToAi(); - sendVideoInput.checked = shouldSendVideoToAi(); - sendAudioInput.addEventListener('change', () => { - writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked); - log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`); - }); - sendVideoInput.addEventListener('change', () => { - writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked); - log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`); - }); - recordButton.addEventListener('click', toggleInputRecording); - videoRecordButton.addEventListener('click', toggleVideoRecording); - updateRecordButton(); - updateVideoRecordButton(); - updateSelectedInputLabel(); - updateModeButton(); - - toggle.addEventListener('click', async () => { - if (!isAiTakeover && !assistantInput.value.trim()) { - alert('请先填写助手 ID。'); - return; - } - - if (!isAiTakeover && !ensureUserTracksBeforeConnect()) { - updateModeButton(); - return; - } - - isAiTakeover = !isAiTakeover; - updateModeButton(); - if (isAiTakeover) { - const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim()); - if (connected) await attachAiOutputToBusiness(); - else { - isAiTakeover = false; - updateModeButton(); - } - } else { - await restoreBusinessAudio(); - disconnectAi(); - } - }); - } - - function renderPanelSoon() { - try { renderPanel(); } catch (error) { console.warn('[AI Native Copilot] panel error', error); } - } - - if (document.readyState === 'loading') { - document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true }); - } else { - renderPanelSoon(); - } - requestAnimationFrame(renderVu); - setInterval(refreshVuFromStats, 250); - setTimeout(renderPanelSoon, 1500); - setTimeout(renderPanelSoon, 3000); -})(); diff --git a/scripts/copilot.js b/scripts/copilot.js index c5d53f6..ab9cf9e 100644 --- a/scripts/copilot.js +++ b/scripts/copilot.js @@ -1,5 +1,5 @@ // ==UserScript== -// @name 12345 AI 协同中台 (WebRTC 洗流桥接版) +// @name 12345 AI 协同中台 // @namespace http://tampermonkey.net/ // @version 7.5 // @description 引入 PCM ScriptProcessor 强制软解防丢包Bug；增加真实洗流耳返监听功能，彻底解决远端触发失败悬案 @@ -23,6 +23,8 @@ } const origAddEventListener = RTCPeerConnection.prototype.addEventListener; const origSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; + const origAddTrack = RTCPeerConnection.prototype.addTrack; + const origAddTransceiver = RTCPeerConnection.prototype.addTransceiver; const origAddStream = RTCPeerConnection.prototype.addStream; const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); @@ -43,6 +45,13 @@ let audioSender = null; let videoSender = null; let analyserNode = null; + let businessAudioSender = null; + let businessOriginalAudioTrack = null; + let aiRemoteAudioTrack = null; + let aiTrackAttachedToBusiness = false; + let aiInputMode = 'washed'; + let directInputHealthTimer = null; + let directInputTrackId = null; let isAiTakeover = false; let isMonitoring = false; @@ -59,6 +68,15 @@ const remoteReceiversByTrackId = new Map(); const receiverStatsMemory = new Map(); const capturedVideoElementStreams = new WeakMap(); + function createAudioContext() { + const AudioContextClass = window.AudioContext || window.webkitAudioContext; + try { + return new AudioContextClass({ sampleRate: 48000 }); + } catch (e) { + console.warn("48k AudioContext 创建失败，回退浏览器默认采样率:", e); + return new AudioContextClass(); + } + } function isInternalPeerConnection(target) { return !!(target && target.__aiCopilotInternal); @@ -98,10 +116,193 @@ console.log(`[AI Copilot WebRTC] 绑定远端 receiver(${reason})`, track.kind, track.id); } + function rememberBusinessAudioSender(sender, track, reason = 'unknown') { + if (!sender) return; + const senderTrack = track || sender.track; + if (senderTrack && senderTrack.kind !== 'audio') return; + businessAudioSender = sender; + if (senderTrack && senderTrack !== aiRemoteAudioTrack) { + businessOriginalAudioTrack = senderTrack; + } + console.log(`[AI Copilot WebRTC] 绑定业务通话 audio sender(${reason})`, senderTrack?.id || 'no-track-yet'); + if (isAiTakeover && aiRemoteAudioTrack) attachAiTrackToBusiness(); + } + + function wrapBusinessSender(sender) { + if (!sender || sender.__aiCopilotWrappedSender) return sender; + const originalReplaceTrack = sender.replaceTrack?.bind(sender); + if (!originalReplaceTrack) return sender; + + sender.replaceTrack = async function(track) { + const ret = await originalReplaceTrack(track); + if (track && track.kind === 'audio' && track !== aiRemoteAudioTrack && !isAiTakeover) { + rememberBusinessAudioSender(sender, track, 'sender-replaceTrack'); + } + return ret; + }; + sender.__aiCopilotWrappedSender = true; + return sender; + } + + async function attachAiTrackToBusiness() { + if (!isAiTakeover || !businessAudioSender || !aiRemoteAudioTrack || aiRemoteAudioTrack.readyState !== 'live') return false; + try { + await businessAudioSender.replaceTrack(aiRemoteAudioTrack); + aiTrackAttachedToBusiness = true; + appendLog("🔁 已将 AI 原生音轨直接替换到业务通话上行", true); + return true; + } catch (e) { + aiTrackAttachedToBusiness = false; + appendLog(`⚠️ AI 原生音轨替换业务 sender 失败: ${e.message || e}`); + return false; + } + } + + async function restoreBusinessAudioTrack() { + if (!businessAudioSender || !aiTrackAttachedToBusiness) return; + try { + await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null); + appendLog("👨‍💼 已恢复业务通话原始音轨", true); + } catch (e) { + appendLog(`⚠️ 恢复业务音轨失败: ${e.message || e}`); + } finally { + aiTrackAttachedToBusiness = false; + } + } + + async function outboundAudioBytes(sender) { + if (!sender?.getStats) return null; + try { + let total = 0; + const report = await sender.getStats(); + report.forEach(stat => { + const statKind = stat.kind || stat.mediaType; + if (stat.type === 'outbound-rtp' && statKind === 'audio' && typeof stat.bytesSent === 'number') { + total += stat.bytesSent; + } + }); + return total; + } catch (e) { + return null; + } + } + + function aiPeerReadyForMedia() { + if (!pc || !pc.remoteDescription) return false; + return ( + pc.connectionState === 'connected' + || pc.iceConnectionState === 'connected' + || pc.iceConnectionState === 'completed' + ); + } + + function aiPeerStateLabel() { + if (!pc) return 'pc=null'; + return `conn=${pc.connectionState},ice=${pc.iceConnectionState},signaling=${pc.signalingState},remote=${pc.remoteDescription?.type || 'none'}`; + } + + function waitForAiPeerMediaReady(timeoutMs = 10000) { + if (aiPeerReadyForMedia()) return Promise.resolve(true); + return new Promise((resolve) => { + const startedAt = Date.now(); + const timer = setInterval(() => { + if (aiPeerReadyForMedia()) { + clearInterval(timer); + resolve(true); + } else if (Date.now() - startedAt >= timeoutMs) { + clearInterval(timer); + resolve(false); + } + }, 250); + }); + } + + function disconnectWashedInputGraph() { + if (currentMeterSourceNode) { + try { currentMeterSourceNode.disconnect(); } catch(e) {} + currentMeterSourceNode = null; + } + if (currentVadBoostGainNode) { + try { currentVadBoostGainNode.disconnect(); } catch(e) {} + currentVadBoostGainNode = null; + } + if (pcmForcerNode) { + try { pcmForcerNode.disconnect(); } catch(e) {} + } + } + + async function fallbackToWashedAiInput(track, reason = 'fallback') { + if (!track || track.readyState !== 'live') return; + if (directInputHealthTimer) { + clearTimeout(directInputHealthTimer); + directInputHealthTimer = null; + } + directInputTrackId = null; + if (audioSender && washedUploadTrack && audioSender.track !== washedUploadTrack) { + try { + await audioSender.replaceTrack(washedUploadTrack); + } catch (e) { + appendLog(`⚠️ AI 输入切回洗流轨失败: ${e.message || e}`); + } + } + aiInputMode = 'washed'; + appendLog(`🧯 AI 输入切回洗流链路(${reason})`, true); + ensureVuMeterForTrack(track, reason); + } + + async function tryDirectAiInput(track, reason = 'direct') { + if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return false; + if (!audioSender) { + pendingBridgeAudioTrack = track; + return false; + } + if (aiInputMode === 'direct' && directInputTrackId === track.id && audioSender.track === track) return true; + + try { + if (directInputHealthTimer) clearTimeout(directInputHealthTimer); + await audioSender.replaceTrack(track); + aiInputMode = 'direct'; + directInputTrackId = track.id; + disconnectWashedInputGraph(); + appendLog(`🚀 AI 输入直连远端原生音轨(${reason}); track=${track.id}, ${aiPeerStateLabel()}`, true); + + directInputHealthTimer = setTimeout(async () => { + if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return; + appendLog(`🩺 等待 AI WebRTC 媒体连接后检查直连输入: ${aiPeerStateLabel()}`); + const ready = await waitForAiPeerMediaReady(10000); + if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return; + if (!ready) { + appendLog(`⚠️ AI WebRTC 媒体连接超时，直连输入无法验证: ${aiPeerStateLabel()}`); + await fallbackToWashedAiInput(track, 'direct-ai-pc-not-ready'); + return; + } + const beforeHealthBytes = await outboundAudioBytes(audioSender); + await new Promise(resolve => setTimeout(resolve, 2500)); + if (aiInputMode !== 'direct' || directInputTrackId !== track.id || audioSender?.track !== track) return; + const afterBytes = await outboundAudioBytes(audioSender); + appendLog(`🩺 直连输入发包检查: before=${beforeHealthBytes}, after=${afterBytes}, ${aiPeerStateLabel()}`); + if ( + afterBytes !== null + && ( + (beforeHealthBytes !== null && afterBytes <= beforeHealthBytes) + || (beforeHealthBytes === null && afterBytes === 0) + ) + ) { + await fallbackToWashedAiInput(track, 'direct-no-outbound-audio'); + } + }, 500); + return true; + } catch (e) { + appendLog(`⚠️ AI 输入直连失败，改用洗流: ${e.message || e}`); + await fallbackToWashedAiInput(track, 'direct-replace-failed'); + return false; + } + } + function initMeterAudioCtx() { if (meterAudioCtx) return; try { - meterAudioCtx = new (window.AudioContext || window.webkitAudioContext)(); + meterAudioCtx = createAudioContext(); meterKeepAliveGainNode = meterAudioCtx.createGain(); meterKeepAliveGainNode.gain.value = 0; @@ -134,14 +335,23 @@ } } - function bridgeRemoteAudioTrack(track, reason = 'scan') { + async function bridgeRemoteAudioTrack(track, reason = 'scan') { if (!track || track.readyState !== 'live' || window.__ignoredTrackIds.has(track.id)) return; - if (track === currentStolenAudioTrack && currentMeterSourceNode) return; + if ( + track === currentStolenAudioTrack + && ( + (aiInputMode === 'direct' && audioSender?.track === track) + || (aiInputMode === 'washed' && currentMeterSourceNode) + ) + ) return; pendingBridgeAudioTrack = track; - appendLog(`🎙️ 锁定远端发声轨(${reason})，接入混流总线`, true); + appendLog(`🎙️ 锁定远端发声轨(${reason})，尝试直连 AI 输入`, true); currentStolenAudioTrack = track; - ensureVuMeterForTrack(track, reason); + const directOk = await tryDirectAiInput(track, reason); + if (!directOk && audioSender) { + await fallbackToWashedAiInput(track, `${reason}-fallback`); + } } function capturePlayableElementVideoStream(el) { @@ -196,7 +406,7 @@ // 2. AGC 放大增益 (击穿 VAD 阈值) const vadBoostGain = meterAudioCtx.createGain(); - vadBoostGain.gain.value = 8.0; // 放大 8 倍 + vadBoostGain.gain.value = 2.0; // fallback 洗流仅轻度放大，避免底噪/回声被过度抬高 currentVadBoostGainNode = vadBoostGain; currentMeterSourceNode.connect(vadBoostGain); @@ -377,6 +587,28 @@ return origAddEventListener.call(this, type, listener, options); }; + RTCPeerConnection.prototype.addTrack = function(track, ...streams) { + const sender = origAddTrack.call(this, track, ...streams); + if (!isInternalPeerConnection(this) && track?.kind === 'audio') { + rememberBusinessAudioSender(wrapBusinessSender(sender), track, 'pc-addTrack'); + } + return sender; + }; + + if (origAddTransceiver) { + RTCPeerConnection.prototype.addTransceiver = function(trackOrKind, init) { + const transceiver = origAddTransceiver.call(this, trackOrKind, init); + if (!isInternalPeerConnection(this)) { + const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : init?.sendEncodings ? transceiver.sender?.track : null; + const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; + if (kind === 'audio') { + rememberBusinessAudioSender(wrapBusinessSender(transceiver.sender), track || transceiver.sender?.track, 'pc-addTransceiver'); + } + } + return transceiver; + }; + } + try { const onTrackDescriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { @@ -526,7 +758,7 @@ } bridgeRemoteVideoTrack(foundVideoTrack); - bridgeRemoteAudioTrack(foundAudioTrack, 'scan'); + bridgeRemoteAudioTrack(foundAudioTrack, 'scan').catch(e => console.warn(e)); }, 1000); // ========================================== @@ -544,39 +776,20 @@ realStream.__internal_bypass = true; realStream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id)); - audioCtx = new (window.AudioContext || window.webkitAudioContext)(); - - const mixedDest = audioCtx.createMediaStreamDestination(); - mixedDest.stream.__internal_bypass = true; - mixedDest.stream.getTracks().forEach(t => window.__ignoredTrackIds.add(t.id)); - - if (realStream.getAudioTracks().length > 0) { - const localMicSource = audioCtx.createMediaStreamSource(realStream); - - micGainNode = audioCtx.createGain(); - micGainNode.gain.value = isAiTakeover ? 0 : 1.0; - localMicSource.connect(micGainNode); - micGainNode.connect(mixedDest); - } - - aiOutGainNode = audioCtx.createGain(); - aiOutGainNode.gain.value = isAiTakeover ? 1.0 : 0; - aiOutGainNode.connect(mixedDest); + audioCtx = createAudioContext(); + micGainNode = null; + aiOutGainNode = null; aiLocalGainNode = audioCtx.createGain(); aiLocalGainNode.gain.value = isAiTakeover ? 1.0 : 0; aiLocalGainNode.connect(audioCtx.destination); - if (constraints.video && realStream.getVideoTracks().length > 0) { - mixedDest.stream.addTrack(realStream.getVideoTracks()[0]); - } - if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') { - bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending'); + bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'pending').catch(e => console.warn(e)); } renderControlPanelSoon(); - return mixedDest.stream; + return realStream; } catch (err) { console.error("🚨 拦截失败:", err); return originalGetUserMedia(constraints); @@ -598,6 +811,9 @@ if(statusEl) statusEl.textContent = "🟡 协商中..."; appendLog("🔄 正在建立 WebRTC 连接..."); + pc.onconnectionstatechange = () => appendLog(`🔌 AI PC connectionState=${pc.connectionState}`); + pc.oniceconnectionstatechange = () => appendLog(`🧊 AI PC iceConnectionState=${pc.iceConnectionState}`); + initMeterAudioCtx(); const activeStream = new MediaStream([washedUploadTrack]); const audioTransceiver = pc.addTransceiver(washedUploadTrack, { @@ -605,7 +821,10 @@ streams: [activeStream] }); audioSender = audioTransceiver.sender; - appendLog("🎤 已挂载洗流音频管线 (附带 PCM 强刷保障)", true); + appendLog("🎤 已挂载 AI 输入音频通道 (优先直连，失败回退洗流)", true); + if (pendingBridgeAudioTrack && pendingBridgeAudioTrack.readyState === 'live') { + bridgeRemoteAudioTrack(pendingBridgeAudioTrack, 'sender-ready').catch(e => console.warn(e)); + } const videoTransceiver = pc.addTransceiver('video', { direction: 'sendonly' }); videoSender = videoTransceiver.sender; @@ -615,16 +834,17 @@ pc.ontrack = (e) => { if (e.track.kind === 'audio' && audioCtx) { - appendLog("🎧 接收到原生 AI 语音流，接入本地拓扑", true); + appendLog("🎧 接收到原生 AI 语音流，准备直连业务通话", true); window.__ignoredTrackIds.add(e.track.id); + aiRemoteAudioTrack = e.track; const aiStream = new MediaStream([e.track]); aiStream.__internal_bypass = true; const aiStreamSource = audioCtx.createMediaStreamSource(aiStream); - aiStreamSource.connect(aiOutGainNode); - aiStreamSource.connect(aiLocalGainNode); + if (aiLocalGainNode) aiStreamSource.connect(aiLocalGainNode); window.__aiStreamSource = aiStreamSource; + attachAiTrackToBusiness(); } }; @@ -715,6 +935,12 @@ } function disconnectWebRTC() { + if (directInputHealthTimer) { + clearTimeout(directInputHealthTimer); + directInputHealthTimer = null; + } + aiInputMode = 'washed'; + directInputTrackId = null; if (dc) { try { dc.close(); } catch(e){} dc = null; @@ -758,7 +984,7 @@
- +
@@ -891,7 +1117,7 @@ fileInput.click(); }); - btnToggleTakeover.addEventListener('click', () => { + btnToggleTakeover.addEventListener('click', async () => { if (!audioCtx) { alert("音频路由未初始化！请先在网页内加入会议。"); return; @@ -907,8 +1133,6 @@ const now = audioCtx.currentTime; if (isAiTakeover) { - if(micGainNode) micGainNode.gain.setValueAtTime(0, now); - if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(1.0, now); if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(1.0, now); btnToggleTakeover.textContent = "🤖 AI 托管中 (点击切回人工)"; @@ -921,10 +1145,13 @@ } connectWebRTC(urlInput.value, asstId); } + if (!businessAudioSender) { + appendLog("⚠️ 尚未捕获业务通话 audio sender，等待网页建立上行轨道"); + } + await attachAiTrackToBusiness(); } else { - if(micGainNode) micGainNode.gain.setValueAtTime(1.0, now); - if(aiOutGainNode) aiOutGainNode.gain.setValueAtTime(0, now); if(aiLocalGainNode) aiLocalGainNode.gain.setValueAtTime(0, now); + await restoreBusinessAudioTrack(); btnToggleTakeover.textContent = "👨‍💼 人工通话中 (点击 AI 托管)"; btnToggleTakeover.style.background = "#10b981";