// ==UserScript== // @name 12345 AI Native Track Copilot // @namespace http://tampermonkey.net/ // @version 1.0 // @description Native-track only bridge: no WebAudio washing, no captureStream, no MediaStreamDestination. // @author You // @match *://yhy-yw.22sj.cn:*/* // @grant none // @run-at document-start // ==/UserScript== (function () { 'use strict'; const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice'; const AUDIO_UPLINK_STORAGE_KEY = 'native-copilot-send-audio'; const VIDEO_UPLINK_STORAGE_KEY = 'native-copilot-send-video'; const originalGetUserMedia = navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices); const originalAddEventListener = RTCPeerConnection.prototype.addEventListener; const originalAddTrack = RTCPeerConnection.prototype.addTrack; const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver; const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription; const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject'); const remoteStreams = new Set(); const ignoredTrackIds = new Set(); const audioCandidates = new Map(); const videoCandidates = new Map(); const businessSenderCandidates = new Map(); const localCaptureTrackIds = new Set(); window.__aiNativeCopilotReceivers = window.__aiNativeCopilotReceivers || new Set(); let aiPc = null; let aiWs = null; let aiDc = null; let aiAudioSender = null; let aiVideoSender = null; let businessAudioSender = null; let businessOriginalAudioTrack = null; let remoteUserAudioTrack = null; let remoteUserVideoTrack = null; let aiResponseAudioTrack = null; let remoteUserAudioLocked = false; let remoteUserVideoLocked = false; let isAiTakeover = false; let aiTrackAttachedToBusiness = false; let inputVuLevel = 0; let aiOutVuLevel = 0; let aiInputHealthTimer = null; let aiOutputHealthTimer = null; let assistantLine = null; let assistantBuffer = ''; let inputRecorder = null; let inputRecordingChunks = []; let inputRecordingTrack = null; let inputRecordingStartedAt = 0; let videoRecorder = null; let videoRecordingChunks = []; let videoRecordingTrack = null; let videoRecordingStartedAt = 0; const receiverStatsMemory = new Map(); const senderStatsMemory = new Map(); const inboundStatsMemory = new Map(); const outboundStatsMemory = new Map(); const businessOutboundStats = { bytesSent: 0, packetsSent: 0, bytesDelta: 0, packetsDelta: 0, }; function log(message) { console.log(`[AI Native Copilot] ${message}`); } function readBoolSetting(key, fallback) { try { const value = localStorage.getItem(key); if (value === null) return fallback; return value === 'true'; } catch { return fallback; } } function writeBoolSetting(key, value) { try { localStorage.setItem(key, value ? 'true' : 'false'); } catch { // Keep the live checkbox state even if storage is unavailable. } } function shouldSendAudioToAi() { return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true); } function shouldSendVideoToAi() { return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true); } function appendChatLine(role, text, important = false) { const box = document.getElementById('native-copilot-log'); if (!box) return; if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; const row = document.createElement('div'); row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`; row.style.marginBottom = '4px'; if (important) row.style.color = '#fcd34d'; box.appendChild(row); while (box.children.length > 120) box.removeChild(box.firstChild); box.scrollTop = box.scrollHeight; return row; } function startAssistantLine() { assistantBuffer = ''; assistantLine = null; } function appendAssistantDelta(delta) { if (!delta) return; const box = document.getElementById('native-copilot-log'); if (!box) { console.log(`[AI Native Copilot] AI: ${delta}`); return; } if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = ''; if (!assistantLine) assistantLine = appendChatLine('AI', ''); assistantBuffer += delta; assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`; box.scrollTop = box.scrollHeight; } function finishAssistantLine() { assistantLine = null; assistantBuffer = ''; } function logUserTranscript(text) { if (!text?.trim()) return; finishAssistantLine(); appendChatLine('用户', text.trim(), true); } function recorderMimeType() { if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus'; if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm'; return ''; } function recordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `native-ai-input-${trackPart}-${stamp}.webm`; } function videoRecordingFileName(track) { const stamp = new Date().toISOString().replace(/[:.]/g, '-'); const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track'; return `native-ai-video-${trackPart}-${stamp}.webm`; } function updateRecordButton() { const button = document.getElementById('native-copilot-record'); if (!button) return; const recording = inputRecorder?.state === 'recording'; button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入'; button.style.background = recording ? '#dc2626' : '#2563eb'; } function videoRecorderMimeType() { if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8'; if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm'; return ''; } function updateVideoRecordButton() { const button = document.getElementById('native-copilot-record-video'); if (!button) return; const recording = videoRecorder?.state === 'recording'; button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频'; button.style.background = recording ? '#dc2626' : '#475569'; } async function startInputRecording() { if (inputRecorder?.state === 'recording') return; chooseRemoteAudioTrack('record'); if (!isUsableTrack(remoteUserAudioTrack)) { appendChatLine('录音', '没有可录制的 AI 输入音轨', true); return; } if (!window.MediaRecorder) { appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true); return; } inputRecordingChunks = []; inputRecordingTrack = remoteUserAudioTrack; inputRecordingStartedAt = Date.now(); const stream = new MediaStream([inputRecordingTrack]); const mimeType = recorderMimeType(); inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); inputRecorder.ondataavailable = (event) => { if (event.data?.size > 0) inputRecordingChunks.push(event.data); }; inputRecorder.onstop = () => { const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = recordingFileName(inputRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录音', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); inputRecorder = null; inputRecordingChunks = []; inputRecordingTrack = null; updateRecordButton(); }; inputRecorder.start(250); appendChatLine('录音', `开始录制 AI 输入:${describeTrack(inputRecordingTrack)}`, true); updateRecordButton(); } function stopInputRecording() { if (inputRecorder?.state === 'recording') inputRecorder.stop(); } function toggleInputRecording() { if (inputRecorder?.state === 'recording') stopInputRecording(); else startInputRecording(); } async function startVideoRecording() { if (videoRecorder?.state === 'recording') return; chooseRemoteVideoTrack('record-video'); if (!remoteUserVideoLocked || !isUsableTrack(remoteUserVideoTrack)) { appendChatLine('录制视频', '没有可录制的上半部用户视频轨', true); return; } if (!window.MediaRecorder) { appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true); return; } videoRecordingChunks = []; videoRecordingTrack = remoteUserVideoTrack; videoRecordingStartedAt = Date.now(); const stream = new MediaStream([videoRecordingTrack]); const mimeType = videoRecorderMimeType(); videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined); videoRecorder.ondataavailable = (event) => { if (event.data?.size > 0) videoRecordingChunks.push(event.data); }; videoRecorder.onstop = () => { const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.href = url; link.download = videoRecordingFileName(videoRecordingTrack); document.body.appendChild(link); link.click(); link.remove(); setTimeout(() => URL.revokeObjectURL(url), 30000); const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1); appendChatLine('录制视频', `已保存 ${link.download}(${seconds}s,${(blob.size / 1024).toFixed(1)} KB)`, true); videoRecorder = null; videoRecordingChunks = []; videoRecordingTrack = null; updateVideoRecordButton(); }; videoRecorder.start(250); appendChatLine('录制视频', `开始录制 AI 视频:${describeTrack(videoRecordingTrack)}`, true); updateVideoRecordButton(); } function stopVideoRecording() { if (videoRecorder?.state === 'recording') videoRecorder.stop(); } function toggleVideoRecording() { if (videoRecorder?.state === 'recording') stopVideoRecording(); else startVideoRecording(); } function generatePcId() { const bytes = new Uint8Array(16); crypto.getRandomValues(bytes); return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join(''); } function isInternalPc(pc) { return Boolean(pc?.__aiNativeCopilotInternal); } function isUsableTrack(track) { return track && track.readyState === 'live' && !ignoredTrackIds.has(track.id); } function isLiveTrack(track) { return track && track.readyState === 'live'; } function candidateFor(track, candidates) { let candidate = candidates.get(track.id); if (!candidate) { candidate = { track, receiver: null, reasons: new Set(), elements: new Set(), firstSeenAt: Date.now(), lastSeenAt: 0, lastPacketAt: 0, bytesReceived: 0, packetsReceived: 0, bytesDelta: 0, packetsDelta: 0, level: 0, score: 0, logged: false, }; candidates.set(track.id, candidate); } candidate.track = track; candidate.lastSeenAt = Date.now(); return candidate; } function businessSenderCandidateFor(sender, track, reason) { if (!sender) return null; const senderTrack = track || sender.track; if (senderTrack && senderTrack.kind !== 'audio') return null; const id = sender.__aiNativeCopilotBusinessSenderId || `business-sender-${businessSenderCandidates.size + 1}`; sender.__aiNativeCopilotBusinessSenderId = id; let candidate = businessSenderCandidates.get(id); if (!candidate) { candidate = { id, sender, track: senderTrack || null, originalTrack: senderTrack || null, sources: new Set(), bytesSent: 0, packetsSent: 0, bytesDelta: 0, packetsDelta: 0, score: 0, logged: false, }; businessSenderCandidates.set(id, candidate); } candidate.sender = sender; candidate.track = senderTrack || sender.track || candidate.track; if (!isAiTakeover && senderTrack && senderTrack !== aiResponseAudioTrack) { candidate.originalTrack = senderTrack; } candidate.sources.add(reason); wrapBusinessSender(sender); return candidate; } function findReceiverForTrack(track) { if (!track) return null; return Array.from(window.__aiNativeCopilotReceivers || []) .find((receiver) => receiver.track === track) || null; } function rememberReceiver(receiver, reason) { if (!receiver?.track || isInternalPc(receiver.transport)) return; window.__aiNativeCopilotReceivers.add(receiver); const track = receiver.track; if (!isUsableTrack(track)) return; if (track.kind === 'audio') { const candidate = candidateFor(track, audioCandidates); candidate.receiver = receiver; candidate.reasons.add(reason); } else if (track.kind === 'video') { const candidate = candidateFor(track, videoCandidates); candidate.receiver = receiver; candidate.reasons.add(reason); chooseRemoteVideoTrack(reason); } } function setVuWidth(id, level) { const el = document.getElementById(id); if (!el) return; el.style.width = `${Math.min(100, level * 2500)}%`; } function renderVu() { inputVuLevel *= 0.88; aiOutVuLevel *= 0.88; setVuWidth('native-copilot-input-vu', inputVuLevel); setVuWidth('native-copilot-ai-vu', aiOutVuLevel); requestAnimationFrame(renderVu); } function levelFromStats(stat, memory) { let level = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0; if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') { const prev = memory.get(stat.id); memory.set(stat.id, { energy: stat.totalAudioEnergy, duration: stat.totalSamplesDuration, }); if (prev) { const energyDelta = stat.totalAudioEnergy - prev.energy; const durationDelta = stat.totalSamplesDuration - prev.duration; if (energyDelta > 0 && durationDelta > 0) { level = Math.max(level, Math.sqrt(energyDelta / durationDelta)); } } } return level; } async function refreshVuFromStats() { try { if (!(await checkLockedUserTracks('stats-before'))) return; await refreshAudioCandidateStats(); chooseRemoteAudioTrack('stats'); await refreshBusinessSenderStats(); chooseBusinessAudioSender('stats'); await checkLockedUserTracks('stats-after'); } catch (error) { // Stats APIs are best-effort across browsers. } } async function refreshBusinessSenderStats() { for (const candidate of businessSenderCandidates.values()) { const sender = candidate.sender; if (!sender?.getStats) continue; candidate.track = sender.track || candidate.track; const report = await sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'outbound-rtp' || kind !== 'audio') return; const key = `${candidate.id}:${stat.id}`; const prev = outboundStatsMemory.get(key); const bytes = typeof stat.bytesSent === 'number' ? stat.bytesSent : 0; const packets = typeof stat.packetsSent === 'number' ? stat.packetsSent : 0; candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; candidate.bytesSent = bytes; candidate.packetsSent = packets; outboundStatsMemory.set(key, { bytes, packets }); if (candidate.sender === businessAudioSender) { businessOutboundStats.bytesDelta = candidate.bytesDelta; businessOutboundStats.packetsDelta = candidate.packetsDelta; businessOutboundStats.bytesSent = candidate.bytesSent; businessOutboundStats.packetsSent = candidate.packetsSent; aiOutVuLevel = Math.max( aiOutVuLevel, levelFromStats(stat, senderStatsMemory), outputVuLevelFromStats() ); } }); } } async function refreshAudioCandidateStats() { for (const candidate of audioCandidates.values()) { if (!isUsableTrack(candidate.track)) continue; const receiver = candidate.receiver || findReceiverForTrack(candidate.track); if (!receiver?.getStats) continue; candidate.receiver = receiver; const report = await receiver.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type !== 'inbound-rtp' || kind !== 'audio') return; const prev = inboundStatsMemory.get(stat.id); const bytes = typeof stat.bytesReceived === 'number' ? stat.bytesReceived : 0; const packets = typeof stat.packetsReceived === 'number' ? stat.packetsReceived : 0; const bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0; const packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0; inboundStatsMemory.set(stat.id, { bytes, packets }); candidate.bytesReceived = bytes; candidate.packetsReceived = packets; candidate.bytesDelta = bytesDelta; candidate.packetsDelta = packetsDelta; candidate.level = levelFromStats(stat, receiverStatsMemory); if (bytesDelta > 0 || packetsDelta > 0 || candidate.level > 0) { candidate.lastPacketAt = Date.now(); } if (candidate.track === remoteUserAudioTrack) { inputVuLevel = Math.max(inputVuLevel, inputVuLevelFromCandidate(candidate)); } }); } } async function outboundAudioBytes(sender) { if (!sender?.getStats) return null; try { let total = 0; const report = await sender.getStats(); report.forEach((stat) => { const kind = stat.kind || stat.mediaType; if (stat.type === 'outbound-rtp' && kind === 'audio' && typeof stat.bytesSent === 'number') { total += stat.bytesSent; } }); return total; } catch { return null; } } function aiReadyForMedia() { if (!aiPc || !aiPc.remoteDescription) return false; return ( aiPc.connectionState === 'connected' || aiPc.iceConnectionState === 'connected' || aiPc.iceConnectionState === 'completed' ); } function aiStateLabel() { if (!aiPc) return 'pc=null'; return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`; } function waitForAiMediaReady(timeoutMs = 10000) { if (aiReadyForMedia()) return Promise.resolve(true); return new Promise((resolve) => { const startedAt = Date.now(); const timer = setInterval(() => { if (aiReadyForMedia()) { clearInterval(timer); resolve(true); } else if (Date.now() - startedAt >= timeoutMs) { clearInterval(timer); resolve(false); } }, 250); }); } function scheduleAiInputHealthCheck(track) { if (aiInputHealthTimer) clearTimeout(aiInputHealthTimer); aiInputHealthTimer = setTimeout(async () => { if (!aiAudioSender || aiAudioSender.track !== track) return; log(`checking native AI input after media connects: ${aiStateLabel()}`); const ready = await waitForAiMediaReady(10000); if (!ready || !aiAudioSender || aiAudioSender.track !== track) { log(`native AI input could not be verified: ${aiStateLabel()}`); return; } const before = await outboundAudioBytes(aiAudioSender); await new Promise((resolve) => setTimeout(resolve, 2500)); if (!aiAudioSender || aiAudioSender.track !== track) return; const after = await outboundAudioBytes(aiAudioSender); log(`native AI input packet check: before=${before}, after=${after}, ${aiStateLabel()}`, true); }, 500); } function describeTrack(track) { if (!track) return 'none'; return `${track.id}${track.muted ? ' muted' : ''}`; } function elementLayoutInfo(element) { if (!element?.getBoundingClientRect || !document.documentElement.contains(element)) return null; const rect = element.getBoundingClientRect(); const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 1; const centerY = rect.top + rect.height / 2; const visibleWidth = Math.max(0, Math.min(rect.right, window.innerWidth || document.documentElement.clientWidth || rect.right) - Math.max(rect.left, 0)); const visibleHeight = Math.max(0, Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0)); return { tag: element.tagName, top: Math.round(rect.top), left: Math.round(rect.left), width: Math.round(rect.width), height: Math.round(rect.height), centerY: Math.round(centerY), upperHalf: centerY <= viewportHeight * 0.55, visibleArea: Math.round(visibleWidth * visibleHeight), paused: Boolean(element.paused), muted: Boolean(element.muted), volume: typeof element.volume === 'number' ? element.volume : null, }; } function candidateLayoutScore(candidate) { let score = 0; for (const element of candidate?.elements || []) { const info = elementLayoutInfo(element); if (!info) continue; if (info.upperHalf) score += 9000; else score -= 2500; if (info.visibleArea > 0) score += Math.min(2500, info.visibleArea / 80); if (info.tag === 'VIDEO') score += 1200; if (info.tag === 'AUDIO') score += 300; if (!info.paused) score += 700; if (!info.muted && (info.volume ?? 1) > 0) score += 500; } return score; } function hasUpperHalfElement(candidate) { return Array.from(candidate?.elements || []).some((element) => elementLayoutInfo(element)?.upperHalf); } function lockedTrackUsable(track, candidates) { const candidate = candidates.get(track?.id); return Boolean(candidate && isUsableTrack(candidate.track) && !candidate.track.muted); } async function handleLockedUserTrackLost(kind, reason) { const message = `${kind === 'audio' ? '远端音频' : '远端视频'}轨道失效(${reason}),已断开 AI`; appendChatLine('选轨', message, true); log(message); if (kind === 'audio') { remoteUserAudioLocked = false; remoteUserAudioTrack = null; } else { remoteUserVideoLocked = false; remoteUserVideoTrack = null; } if (isAiTakeover) { isAiTakeover = false; updateModeButton(); await restoreBusinessAudio(); disconnectAi(); } } async function checkLockedUserTracks(reason) { if (remoteUserAudioLocked && !lockedTrackUsable(remoteUserAudioTrack, audioCandidates)) { await handleLockedUserTrackLost('audio', reason); return false; } if (remoteUserVideoLocked && !lockedTrackUsable(remoteUserVideoTrack, videoCandidates)) { await handleLockedUserTrackLost('video', reason); return false; } return true; } function scoreAudioCandidate(candidate) { if (!candidate || !isUsableTrack(candidate.track)) return -1; const hasRecentPackets = Date.now() - candidate.lastPacketAt < 3000; const hasAudibleLevel = candidate.level >= 0.0015; let score = 0; if (candidate.track.muted) score -= 5000; else score += 800; score += candidateLayoutScore(candidate); if (candidate.receiver) score += 600; if (candidate.bytesReceived > 0) score += 100; if (candidate.bytesDelta > 0) score += 500 + Math.min(900, candidate.bytesDelta * 2); if (candidate.packetsDelta > 0) score += 300 + Math.min(500, candidate.packetsDelta * 8); if (hasRecentPackets) score += 400; if (hasAudibleLevel) score += 5000; score += Math.min(12000, candidate.level * 900000); if (!hasAudibleLevel && hasRecentPackets) score -= 1800; if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 80; return score; } function scoreVideoCandidate(candidate) { if (!candidate || !isUsableTrack(candidate.track)) return -1; let score = 0; if (candidate.track.muted) score -= 1000; else score += 500; if (candidate.receiver) score += 600; score += candidateLayoutScore(candidate); for (const element of candidate.elements || []) { const info = elementLayoutInfo(element); if (!info) continue; if (info.tag === 'VIDEO') score += 2500; if (info.visibleArea > 0) score += Math.min(4000, info.visibleArea / 50); } if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 100; return score; } function inputVuLevelFromCandidate(candidate) { if (!candidate) return 0; return Math.max( candidate.level, Math.min(0.04, candidate.bytesDelta / 16000), Math.min(0.02, candidate.packetsDelta / 800) ); } function updateSelectedInputLabel() { const label = document.getElementById('native-copilot-selected-input'); if (!label) return; const candidate = audioCandidates.get(remoteUserAudioTrack?.id); if (!candidate) { label.textContent = '等待远端音轨'; return; } const layout = Array.from(candidate.elements || []).map(elementLayoutInfo).find(Boolean); const where = layout ? (layout.upperHalf ? 'upper' : 'lower') : 'no-layout'; label.textContent = `${remoteUserAudioTrack.id.slice(0, 8)} ${where} level=${candidate.level.toFixed(4)} score=${Math.round(candidate.score || scoreAudioCandidate(candidate))}`; } function outputVuLevelFromStats() { return Math.max( Math.min(0.12, businessOutboundStats.bytesDelta / 12000), Math.min(0.04, businessOutboundStats.packetsDelta / 500) ); } function isKnownRemoteAudioTrack(track) { return Boolean(track && ( track === remoteUserAudioTrack || audioCandidates.has(track.id) )); } function isAiOutputTrack(track) { return Boolean(track && track === aiResponseAudioTrack); } function scoreBusinessSenderCandidate(candidate) { if (!candidate?.sender) return -1; const track = candidate.sender.track || candidate.track; let score = 0; if (track?.kind === 'audio') score += 500; else if (!track) score -= 1000; if (track?.readyState === 'live') score += 700; if (track && !track.muted) score += 300; if (candidate.sources.has('pc-addTrack')) score += 5000; if (candidate.sources.has('pc-addTransceiver')) score -= 1000; if (track && localCaptureTrackIds.has(track.id)) score += 2500; if (candidate.originalTrack && localCaptureTrackIds.has(candidate.originalTrack.id)) score += 2500; if (isKnownRemoteAudioTrack(track)) score -= 9000; if (candidate.originalTrack && isKnownRemoteAudioTrack(candidate.originalTrack)) score -= 9000; if (isAiOutputTrack(track)) score += 4500; if (candidate.bytesSent > 0) score += 600; if (candidate.bytesDelta > 0) score += 1600 + Math.min(1800, candidate.bytesDelta * 4); if (candidate.packetsDelta > 0) score += 800 + Math.min(900, candidate.packetsDelta * 10); return score; } function chooseBusinessAudioSender(reason) { let best = null; let bestScore = -1; for (const candidate of businessSenderCandidates.values()) { candidate.score = scoreBusinessSenderCandidate(candidate); if (candidate.score > bestScore) { best = candidate; bestScore = candidate.score; } } if (!best || bestScore < 0) return false; if (businessAudioSender === best.sender) return true; businessAudioSender = best.sender; businessOriginalAudioTrack = best.originalTrack || best.sender.track || businessOriginalAudioTrack; log(`selected business audio sender (${reason}): ${best.id}, track=${describeTrack(best.sender.track)}, score=${Math.round(bestScore)}, sources=${Array.from(best.sources).join('+')}`, true); attachAiOutputToBusiness(); return true; } function chooseRemoteAudioTrack(reason) { const lockedAudio = audioCandidates.get(remoteUserAudioTrack?.id); if (remoteUserAudioLocked && lockedAudio && isUsableTrack(lockedAudio.track) && !lockedAudio.track.muted) { updateSelectedInputLabel(); return true; } if (remoteUserAudioLocked) return false; let best = null; let bestScore = -1; for (const candidate of audioCandidates.values()) { candidate.score = scoreAudioCandidate(candidate); if (candidate.score > bestScore) { best = candidate; bestScore = candidate.score; } } if (!best || bestScore < 0) return false; if (remoteUserAudioTrack === best.track) { updateSelectedInputLabel(); return true; } const current = audioCandidates.get(remoteUserAudioTrack?.id); const currentScore = scoreAudioCandidate(current); const currentHealthy = current && isUsableTrack(current.track) && !current.track.muted && (current.level >= 0.0015 || hasUpperHalfElement(current)) && Date.now() - current.lastPacketAt < 3000; if (currentHealthy && bestScore < currentScore + 1800) { updateSelectedInputLabel(); return true; } remoteUserAudioTrack = best.track; remoteUserAudioLocked = hasUpperHalfElement(best); log(`${remoteUserAudioLocked ? 'locked' : 'selected'} remote audio track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, level=${best.level.toFixed(4)}, bytesDelta=${best.bytesDelta}, packetsDelta=${best.packetsDelta}`, true); updateSelectedInputLabel(); attachRemoteInputToAi(); return true; } function chooseRemoteVideoTrack(reason) { const lockedVideo = videoCandidates.get(remoteUserVideoTrack?.id); if (remoteUserVideoLocked && lockedVideo && isUsableTrack(lockedVideo.track) && !lockedVideo.track.muted) { return true; } if (remoteUserVideoLocked) return false; let best = null; let bestScore = -1; for (const candidate of videoCandidates.values()) { candidate.score = scoreVideoCandidate(candidate); if (candidate.score > bestScore) { best = candidate; bestScore = candidate.score; } } if (!best || bestScore < 0) return false; if (remoteUserVideoTrack === best.track) return true; const current = videoCandidates.get(remoteUserVideoTrack?.id); const currentScore = scoreVideoCandidate(current); if (hasUpperHalfElement(current) && bestScore < currentScore + 1800) return true; remoteUserVideoTrack = best.track; remoteUserVideoLocked = hasUpperHalfElement(best); log(`${remoteUserVideoLocked ? 'locked' : 'selected'} remote video track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, layout=${Math.round(candidateLayoutScore(best))}`, true); attachRemoteVideoToAi(); return true; } function ensureUserTracksBeforeConnect() { chooseRemoteAudioTrack('pre-connect'); chooseRemoteVideoTrack('pre-connect'); if (shouldSendAudioToAi() && (!remoteUserAudioLocked || !lockedTrackUsable(remoteUserAudioTrack, audioCandidates))) { appendChatLine('选轨', '未锁定上半部用户音频轨,暂不连接 AI', true); return false; } if (shouldSendVideoToAi() && (!remoteUserVideoLocked || !lockedTrackUsable(remoteUserVideoTrack, videoCandidates))) { appendChatLine('选轨', '未锁定上半部用户视频轨,暂不连接 AI', true); return false; } return true; } function rememberRemoteTrack(track, reason, element = null) { if (!isUsableTrack(track)) return; if (track.kind === 'audio') { const candidate = candidateFor(track, audioCandidates); candidate.receiver = candidate.receiver || findReceiverForTrack(track); candidate.reasons.add(reason); if (element) candidate.elements.add(element); if (!candidate.logged) { candidate.logged = true; log(`captured remote audio candidate (${reason}): ${describeTrack(track)}`, true); } chooseRemoteAudioTrack(reason); return; } if (track.kind === 'video') { const candidate = candidateFor(track, videoCandidates); candidate.receiver = candidate.receiver || findReceiverForTrack(track); candidate.reasons.add(reason); if (element) candidate.elements.add(element); if (!candidate.logged) { candidate.logged = true; log(`captured remote video candidate (${reason}): ${describeTrack(track)}`, true); } chooseRemoteVideoTrack(reason); } } function rememberRemoteStream(stream, reason, element = null) { if (!stream || !(stream instanceof MediaStream)) return; if (stream.__aiNativeCopilotInternal) return; const liveTracks = stream.getTracks().filter(isUsableTrack); if (liveTracks.length === 0) return; remoteStreams.add(stream); for (const track of liveTracks) rememberRemoteTrack(track, reason, element); chooseRemoteVideoTrack(reason); } function rememberBusinessAudioSender(sender, track, reason) { const candidate = businessSenderCandidateFor(sender, track, reason); if (!candidate) return; if (!candidate.logged) { candidate.logged = true; log(`captured business audio sender candidate (${reason}): ${candidate.id}, track=${describeTrack(candidate.track)}`, true); } chooseBusinessAudioSender(reason); } function wrapBusinessSender(sender) { if (!sender || sender.__aiNativeCopilotWrapped) return; const originalReplaceTrack = sender.replaceTrack?.bind(sender); if (!originalReplaceTrack) return; sender.replaceTrack = async function (track) { const result = await originalReplaceTrack(track); const candidate = businessSenderCandidateFor(sender, track, 'replaceTrack'); if (candidate) { candidate.track = track || sender.track || candidate.track; } if (!isAiTakeover && track?.kind === 'audio' && track !== aiResponseAudioTrack) { if (candidate) candidate.originalTrack = track; chooseBusinessAudioSender('replaceTrack'); } return result; }; sender.__aiNativeCopilotWrapped = true; } async function attachRemoteInputToAi() { if (!shouldSendAudioToAi()) return false; if (!aiAudioSender || !isUsableTrack(remoteUserAudioTrack)) return false; if (aiAudioSender.track === remoteUserAudioTrack) return true; try { await aiAudioSender.replaceTrack(remoteUserAudioTrack); log(`AI input uses native remote audio track (${remoteUserAudioTrack.id})`, true); scheduleAiInputHealthCheck(remoteUserAudioTrack); return true; } catch (error) { log(`failed to attach native remote audio to AI: ${error.message || error}`); return false; } } async function attachRemoteVideoToAi() { if (!shouldSendVideoToAi()) return false; if (!aiVideoSender || !isUsableTrack(remoteUserVideoTrack)) return false; if (aiVideoSender.track === remoteUserVideoTrack) return true; try { await aiVideoSender.replaceTrack(remoteUserVideoTrack); log('AI video uses native remote video track', true); return true; } catch (error) { log(`failed to attach native remote video to AI: ${error.message || error}`); return false; } } async function attachAiOutputToBusiness() { chooseBusinessAudioSender('before-ai-output'); if (!isAiTakeover || !businessAudioSender || !isLiveTrack(aiResponseAudioTrack)) { log(`AI output waiting: takeover=${isAiTakeover}, businessSender=${Boolean(businessAudioSender)}, aiTrack=${describeTrack(aiResponseAudioTrack)}`); return false; } if (businessAudioSender.track === aiResponseAudioTrack) return true; try { await businessAudioSender.replaceTrack(aiResponseAudioTrack); aiTrackAttachedToBusiness = true; log(`business call now uses native AI response audio track (${aiResponseAudioTrack.id})`, true); scheduleAiOutputHealthCheck(aiResponseAudioTrack); return true; } catch (error) { aiTrackAttachedToBusiness = false; log(`failed to attach AI audio to business call: ${error.message || error}`); return false; } } function scheduleAiOutputHealthCheck(track) { if (aiOutputHealthTimer) clearTimeout(aiOutputHealthTimer); aiOutputHealthTimer = setTimeout(async () => { if (!businessAudioSender || businessAudioSender.track !== track) { log(`AI output check skipped: senderTrack=${describeTrack(businessAudioSender?.track)}, aiTrack=${describeTrack(track)}`); return; } const before = await outboundAudioBytes(businessAudioSender); await new Promise((resolve) => setTimeout(resolve, 2500)); if (!businessAudioSender || businessAudioSender.track !== track) return; const after = await outboundAudioBytes(businessAudioSender); log(`AI output packet check: before=${before}, after=${after}, senderTrack=${describeTrack(businessAudioSender.track)}`, true); }, 800); } async function restoreBusinessAudio() { if (!businessAudioSender || !aiTrackAttachedToBusiness) return; try { await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null); log('business call restored original audio track', true); } catch (error) { log(`failed to restore business audio track: ${error.message || error}`); } finally { aiTrackAttachedToBusiness = false; } } function wrapTrackListener(listener) { if (!listener || listener.__aiNativeCopilotWrapped) return listener; const wrapped = function (event) { if (!isInternalPc(this) && event?.track) { rememberReceiver(event.receiver, 'pc-track'); rememberRemoteTrack(event.track, 'pc-track'); for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream'); } if (typeof listener === 'function') return listener.apply(this, arguments); if (listener?.handleEvent) return listener.handleEvent(event); return undefined; }; wrapped.__aiNativeCopilotWrapped = true; return wrapped; } RTCPeerConnection.prototype.addEventListener = function (type, listener, options) { if (type === 'track') { return originalAddEventListener.call(this, type, wrapTrackListener(listener), options); } return originalAddEventListener.call(this, type, listener, options); }; try { const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack'); Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', { configurable: true, enumerable: true, get() { return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeCopilotOnTrack || null; }, set(listener) { const wrapped = wrapTrackListener(listener); this.__aiNativeCopilotOnTrack = wrapped; if (descriptor?.set) descriptor.set.call(this, wrapped); else this.addEventListener('track', wrapped); }, }); } catch (error) { console.warn('[AI Native Copilot] failed to wrap ontrack', error); } RTCPeerConnection.prototype.addTrack = function (track, ...streams) { const sender = originalAddTrack.call(this, track, ...streams); if (!isInternalPc(this) && track?.kind === 'audio') { rememberBusinessAudioSender(sender, track, 'pc-addTrack'); } return sender; }; if (originalAddTransceiver) { RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) { const transceiver = originalAddTransceiver.call(this, trackOrKind, init); if (!isInternalPc(this)) { const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind; if (kind === 'audio') { const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track; rememberBusinessAudioSender(transceiver.sender, track, 'pc-addTransceiver'); } } return transceiver; }; } RTCPeerConnection.prototype.setRemoteDescription = function () { const result = originalSetRemoteDescription.apply(this, arguments); if (!isInternalPc(this)) { Promise.resolve(result).then(() => { try { for (const receiver of this.getReceivers?.() || []) { rememberReceiver(receiver, 'receiver-after-srd'); rememberRemoteTrack(receiver.track, 'receiver-after-srd'); } for (const sender of this.getSenders?.() || []) { rememberBusinessAudioSender(sender, sender.track, 'sender-after-srd'); } for (const stream of this.getRemoteStreams?.() || []) { rememberRemoteStream(stream, 'remote-stream-after-srd'); } } catch (error) { console.warn('[AI Native Copilot] failed to scan receivers', error); } }); } return result; }; if (mediaSrcObjectDescriptor?.set) { Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', { configurable: true, enumerable: mediaSrcObjectDescriptor.enumerable, get() { return mediaSrcObjectDescriptor.get.call(this); }, set(stream) { if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiNativeCopilotElement) { rememberRemoteStream(stream, 'media-srcObject', this); } return mediaSrcObjectDescriptor.set.call(this, stream); }, }); } if (originalGetUserMedia) { navigator.mediaDevices.getUserMedia = async function (constraints) { const stream = await originalGetUserMedia(constraints); for (const track of stream.getAudioTracks()) { localCaptureTrackIds.add(track.id); } chooseBusinessAudioSender('getUserMedia'); renderPanelSoon(); return stream; }; } setInterval(async () => { if (!(await checkLockedUserTracks('scan'))) return; for (const element of document.querySelectorAll('audio, video')) { if (element.__aiNativeCopilotElement) continue; rememberRemoteStream(element.srcObject, 'media-scan', element); } for (const stream of Array.from(remoteStreams)) { rememberRemoteStream(stream, 'stream-rescan'); } await checkLockedUserTracks('scan-after'); }, 1000); window.__aiNativeCopilotDump = function () { const audio = Array.from(audioCandidates.values()).map((candidate) => ({ id: candidate.track.id, label: candidate.track.label, selected: candidate.track === remoteUserAudioTrack, enabled: candidate.track.enabled, muted: candidate.track.muted, readyState: candidate.track.readyState, hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)), bytesReceived: candidate.bytesReceived, packetsReceived: candidate.packetsReceived, bytesDelta: candidate.bytesDelta, packetsDelta: candidate.packetsDelta, level: candidate.level, audible: candidate.level >= 0.0015, layoutScore: Math.round(candidateLayoutScore(candidate)), elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean), vuLevel: inputVuLevelFromCandidate(candidate), score: Math.round(scoreAudioCandidate(candidate)), reasons: Array.from(candidate.reasons), })); const video = Array.from(videoCandidates.values()).map((candidate) => ({ id: candidate.track.id, label: candidate.track.label, selected: candidate.track === remoteUserVideoTrack, muted: candidate.track.muted, readyState: candidate.track.readyState, hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)), layoutScore: Math.round(candidateLayoutScore(candidate)), score: Math.round(scoreVideoCandidate(candidate)), elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean), reasons: Array.from(candidate.reasons), })); const businessSenders = Array.from(businessSenderCandidates.values()).map((candidate) => ({ id: candidate.id, selected: candidate.sender === businessAudioSender, track: describeTrack(candidate.sender.track || candidate.track), originalTrack: describeTrack(candidate.originalTrack), isRemoteTrack: isKnownRemoteAudioTrack(candidate.sender.track || candidate.track), isLocalCaptureTrack: Boolean(candidate.sender.track && localCaptureTrackIds.has(candidate.sender.track.id)), bytesSent: candidate.bytesSent, packetsSent: candidate.packetsSent, bytesDelta: candidate.bytesDelta, packetsDelta: candidate.packetsDelta, score: Math.round(scoreBusinessSenderCandidate(candidate)), sources: Array.from(candidate.sources), })); return { selectedAudio: describeTrack(remoteUserAudioTrack), selectedVideo: describeTrack(remoteUserVideoTrack), audioLocked: remoteUserAudioLocked, videoLocked: remoteUserVideoLocked, aiInputTrack: describeTrack(aiAudioSender?.track), aiOutputTrack: describeTrack(aiResponseAudioTrack), businessSenderTrack: describeTrack(businessAudioSender?.track), aiTrackAttachedToBusiness, videoRecording: videoRecorder?.state === 'recording' ? { track: describeTrack(videoRecordingTrack), seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1), chunks: videoRecordingChunks.length, } : null, businessOutboundStats: { ...businessOutboundStats }, aiState: aiStateLabel(), audio, video, businessSenders, }; }; async function connectAi(signalingUrl, assistantId) { if (!ensureUserTracksBeforeConnect()) { setStatus('disconnected'); return false; } disconnectAi(); const pcId = generatePcId(); aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] }); aiPc.__aiNativeCopilotInternal = true; aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`); aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`); aiDc = aiPc.createDataChannel('chat'); aiDc.onopen = () => { setStatus('connected'); log('AI data channel open', true); aiDc.send(JSON.stringify({ type: 'client-ready' })); }; aiDc.onclose = () => setStatus('disconnected'); aiDc.onmessage = (event) => { try { const msg = JSON.parse(event.data); const content = msg.delta || msg.content; if (msg.type === 'transcript') { logUserTranscript(content); } else if (msg.type === 'assistant-text-start') { startAssistantLine(); } else if (msg.type === 'assistant-text-delta') { appendAssistantDelta(content); } else if (msg.type === 'assistant-text-end') { finishAssistantLine(); } else { log(`data channel message: ${msg.type}`); } } catch { appendChatLine('DATA', event.data); } }; const sendAudio = shouldSendAudioToAi(); const audioTransceiver = sendAudio && isUsableTrack(remoteUserAudioTrack) ? aiPc.addTransceiver(remoteUserAudioTrack, { direction: 'sendrecv' }) : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' }); aiAudioSender = audioTransceiver.sender; log(!sendAudio ? 'AI audio uplink disabled by panel setting' : isUsableTrack(remoteUserAudioTrack) ? 'AI input starts with native remote audio track' : 'AI input waiting for native remote audio track', true); const sendVideo = shouldSendVideoToAi(); if (sendVideo) { const videoTransceiver = isUsableTrack(remoteUserVideoTrack) ? aiPc.addTransceiver(remoteUserVideoTrack, { direction: 'sendonly' }) : aiPc.addTransceiver('video', { direction: 'sendonly' }); aiVideoSender = videoTransceiver.sender; if (isUsableTrack(remoteUserVideoTrack)) log('AI video starts with native remote video track', true); } else { aiVideoSender = null; log('AI video uplink disabled by panel setting'); } if (sendAudio && isUsableTrack(remoteUserAudioTrack)) scheduleAiInputHealthCheck(remoteUserAudioTrack); aiPc.ontrack = (event) => { if (event.track.kind !== 'audio') return; ignoredTrackIds.add(event.track.id); aiResponseAudioTrack = event.track; log('received native AI response audio track', true); attachAiOutputToBusiness(); }; aiPc.onicecandidate = (event) => { if (aiWs?.readyState !== WebSocket.OPEN) return; aiWs.send(JSON.stringify({ type: 'ice-candidate', payload: { pc_id: pcId, candidate: event.candidate ? { candidate: event.candidate.candidate, sdpMid: event.candidate.sdpMid, sdpMLineIndex: event.candidate.sdpMLineIndex, } : null, }, })); }; setStatus('connecting'); aiWs = new WebSocket(signalingUrl); aiWs.onopen = async () => { const offer = await aiPc.createOffer(); await aiPc.setLocalDescription(offer); aiWs.send(JSON.stringify({ type: 'offer', payload: { pc_id: pcId, sdp: aiPc.localDescription.sdp, type: aiPc.localDescription.type, assistant_id: assistantId, vision_enabled: sendVideo, }, })); log('sent AI WebRTC offer', true); }; aiWs.onmessage = async (event) => { try { const msg = JSON.parse(event.data); if (msg.type === 'answer') { await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp }); log(`AI WebRTC answer applied; ${aiStateLabel()}`, true); } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) { await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {}); } else if (msg.type === 'error') { throw new Error(msg.payload?.message || 'server error'); } } catch (error) { log(`signaling error: ${error.message || error}`); } }; aiWs.onerror = () => { setStatus('failed'); log('AI signaling websocket error'); }; aiWs.onclose = () => setStatus('disconnected'); attachRemoteInputToAi(); if (sendVideo) attachRemoteVideoToAi(); return true; } function disconnectAi() { if (aiDc) { try { aiDc.close(); } catch {} aiDc = null; } if (aiPc) { try { aiPc.close(); } catch {} aiPc = null; } if (aiWs) { try { aiWs.close(); } catch {} aiWs = null; } aiAudioSender = null; aiVideoSender = null; aiResponseAudioTrack = null; if (inputRecorder?.state === 'recording') stopInputRecording(); if (videoRecorder?.state === 'recording') stopVideoRecording(); setStatus('disconnected'); } function setStatus(status) { const el = document.getElementById('native-copilot-status'); if (!el) return; const labels = { disconnected: '未连接', connecting: '连接中', connected: '已连接', failed: '连接失败', }; el.textContent = labels[status] || status; } function updateModeButton() { const button = document.getElementById('native-copilot-toggle'); if (!button) return; if (isAiTakeover) { button.textContent = '当前:AI 接管|点击切回人工'; button.style.background = '#8b5cf6'; } else { button.textContent = '当前:人工接管|点击切换 AI'; button.style.background = '#10b981'; } } function renderPanel() { if (!document.body || document.getElementById('native-copilot-panel')) return; const panel = document.createElement('div'); panel.id = 'native-copilot-panel'; panel.style.cssText = [ 'position:fixed', 'top:20px', 'left:20px', 'width:360px', 'z-index:999999', 'box-sizing:border-box', 'padding:16px', 'border-radius:10px', 'background:rgba(17,24,39,.96)', 'color:white', 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif', 'box-shadow:0 12px 32px rgba(0,0,0,.35)', ].join(';'); panel.innerHTML = `