diff --git a/scripts/copilot-native.js b/scripts/copilot-native.js
new file mode 100644
index 0000000..c340435
--- /dev/null
+++ b/scripts/copilot-native.js
@@ -0,0 +1,1542 @@
+// ==UserScript==
+// @name 12345 AI Native Track Copilot
+// @namespace http://tampermonkey.net/
+// @version 1.0
+// @description Native-track only bridge: no WebAudio washing, no captureStream, no MediaStreamDestination.
+// @author You
+// @match *://yhy-yw.22sj.cn:*/*
+// @grant none
+// @run-at document-start
+// ==/UserScript==
+
+(function () {
+ 'use strict';
+
+ const DEFAULT_SIGNALING_URL = 'ws://182.92.86.220:8000/ws/voice';
+ const AUDIO_UPLINK_STORAGE_KEY = 'native-copilot-send-audio';
+ const VIDEO_UPLINK_STORAGE_KEY = 'native-copilot-send-video';
+
+ const originalGetUserMedia = navigator.mediaDevices?.getUserMedia?.bind(navigator.mediaDevices);
+ const originalAddEventListener = RTCPeerConnection.prototype.addEventListener;
+ const originalAddTrack = RTCPeerConnection.prototype.addTrack;
+ const originalAddTransceiver = RTCPeerConnection.prototype.addTransceiver;
+ const originalSetRemoteDescription = RTCPeerConnection.prototype.setRemoteDescription;
+ const mediaSrcObjectDescriptor = Object.getOwnPropertyDescriptor(HTMLMediaElement.prototype, 'srcObject');
+
+ const remoteStreams = new Set();
+ const ignoredTrackIds = new Set();
+ const audioCandidates = new Map();
+ const videoCandidates = new Map();
+ const businessSenderCandidates = new Map();
+ const localCaptureTrackIds = new Set();
+ window.__aiNativeCopilotReceivers = window.__aiNativeCopilotReceivers || new Set();
+
+ let aiPc = null;
+ let aiWs = null;
+ let aiDc = null;
+ let aiAudioSender = null;
+ let aiVideoSender = null;
+
+ let businessAudioSender = null;
+ let businessOriginalAudioTrack = null;
+
+ let remoteUserAudioTrack = null;
+ let remoteUserVideoTrack = null;
+ let aiResponseAudioTrack = null;
+ let remoteUserAudioLocked = false;
+ let remoteUserVideoLocked = false;
+
+ let isAiTakeover = false;
+ let aiTrackAttachedToBusiness = false;
+ let inputVuLevel = 0;
+ let aiOutVuLevel = 0;
+ let aiInputHealthTimer = null;
+ let aiOutputHealthTimer = null;
+ let assistantLine = null;
+ let assistantBuffer = '';
+ let inputRecorder = null;
+ let inputRecordingChunks = [];
+ let inputRecordingTrack = null;
+ let inputRecordingStartedAt = 0;
+ let videoRecorder = null;
+ let videoRecordingChunks = [];
+ let videoRecordingTrack = null;
+ let videoRecordingStartedAt = 0;
+
+ const receiverStatsMemory = new Map();
+ const senderStatsMemory = new Map();
+ const inboundStatsMemory = new Map();
+ const outboundStatsMemory = new Map();
+ const businessOutboundStats = {
+ bytesSent: 0,
+ packetsSent: 0,
+ bytesDelta: 0,
+ packetsDelta: 0,
+ };
+
+ function log(message) {
+ console.log(`[AI Native Copilot] ${message}`);
+ }
+
+ function readBoolSetting(key, fallback) {
+ try {
+ const value = localStorage.getItem(key);
+ if (value === null) return fallback;
+ return value === 'true';
+ } catch {
+ return fallback;
+ }
+ }
+
+ function writeBoolSetting(key, value) {
+ try {
+ localStorage.setItem(key, value ? 'true' : 'false');
+ } catch {
+ // Keep the live checkbox state even if storage is unavailable.
+ }
+ }
+
+ function shouldSendAudioToAi() {
+ return readBoolSetting(AUDIO_UPLINK_STORAGE_KEY, true);
+ }
+
+ function shouldSendVideoToAi() {
+ return readBoolSetting(VIDEO_UPLINK_STORAGE_KEY, true);
+ }
+
+ function appendChatLine(role, text, important = false) {
+ const box = document.getElementById('native-copilot-log');
+ if (!box) return;
+ if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
+
+ const row = document.createElement('div');
+ row.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] ${role}: ${text}`;
+ row.style.marginBottom = '4px';
+ if (important) row.style.color = '#fcd34d';
+ box.appendChild(row);
+ while (box.children.length > 120) box.removeChild(box.firstChild);
+ box.scrollTop = box.scrollHeight;
+ return row;
+ }
+
+ function startAssistantLine() {
+ assistantBuffer = '';
+ assistantLine = null;
+ }
+
+ function appendAssistantDelta(delta) {
+ if (!delta) return;
+ const box = document.getElementById('native-copilot-log');
+ if (!box) {
+ console.log(`[AI Native Copilot] AI: ${delta}`);
+ return;
+ }
+ if (box.children.length === 1 && box.children[0].dataset.placeholder) box.innerHTML = '';
+ if (!assistantLine) assistantLine = appendChatLine('AI', '');
+ assistantBuffer += delta;
+ assistantLine.textContent = `[${new Date().toLocaleTimeString([], { hour12: false })}] AI: ${assistantBuffer}`;
+ box.scrollTop = box.scrollHeight;
+ }
+
+ function finishAssistantLine() {
+ assistantLine = null;
+ assistantBuffer = '';
+ }
+
+ function logUserTranscript(text) {
+ if (!text?.trim()) return;
+ finishAssistantLine();
+ appendChatLine('用户', text.trim(), true);
+ }
+
+ function recorderMimeType() {
+ if (window.MediaRecorder?.isTypeSupported?.('audio/webm;codecs=opus')) return 'audio/webm;codecs=opus';
+ if (window.MediaRecorder?.isTypeSupported?.('audio/webm')) return 'audio/webm';
+ return '';
+ }
+
+ function recordingFileName(track) {
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
+ return `native-ai-input-${trackPart}-${stamp}.webm`;
+ }
+
+ function videoRecordingFileName(track) {
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const trackPart = track?.id ? track.id.slice(0, 8) : 'no-track';
+ return `native-ai-video-${trackPart}-${stamp}.webm`;
+ }
+
+ function updateRecordButton() {
+ const button = document.getElementById('native-copilot-record');
+ if (!button) return;
+ const recording = inputRecorder?.state === 'recording';
+ button.textContent = recording ? '停止录制 AI 输入' : '录制 AI 输入';
+ button.style.background = recording ? '#dc2626' : '#2563eb';
+ }
+
+ function videoRecorderMimeType() {
+ if (window.MediaRecorder?.isTypeSupported?.('video/webm;codecs=vp8')) return 'video/webm;codecs=vp8';
+ if (window.MediaRecorder?.isTypeSupported?.('video/webm')) return 'video/webm';
+ return '';
+ }
+
+ function updateVideoRecordButton() {
+ const button = document.getElementById('native-copilot-record-video');
+ if (!button) return;
+ const recording = videoRecorder?.state === 'recording';
+ button.textContent = recording ? '停止录制 AI 视频' : '录制 AI 视频';
+ button.style.background = recording ? '#dc2626' : '#475569';
+ }
+
+ async function startInputRecording() {
+ if (inputRecorder?.state === 'recording') return;
+ chooseRemoteAudioTrack('record');
+ if (!isUsableTrack(remoteUserAudioTrack)) {
+ appendChatLine('录音', '没有可录制的 AI 输入音轨', true);
+ return;
+ }
+ if (!window.MediaRecorder) {
+ appendChatLine('录音', '当前浏览器不支持 MediaRecorder', true);
+ return;
+ }
+
+ inputRecordingChunks = [];
+ inputRecordingTrack = remoteUserAudioTrack;
+ inputRecordingStartedAt = Date.now();
+ const stream = new MediaStream([inputRecordingTrack]);
+ const mimeType = recorderMimeType();
+ inputRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
+ inputRecorder.ondataavailable = (event) => {
+ if (event.data?.size > 0) inputRecordingChunks.push(event.data);
+ };
+ inputRecorder.onstop = () => {
+ const blob = new Blob(inputRecordingChunks, { type: inputRecorder?.mimeType || 'audio/webm' });
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement('a');
+ link.href = url;
+ link.download = recordingFileName(inputRecordingTrack);
+ document.body.appendChild(link);
+ link.click();
+ link.remove();
+ setTimeout(() => URL.revokeObjectURL(url), 30000);
+
+ const seconds = ((Date.now() - inputRecordingStartedAt) / 1000).toFixed(1);
+ appendChatLine('录音', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
+ inputRecorder = null;
+ inputRecordingChunks = [];
+ inputRecordingTrack = null;
+ updateRecordButton();
+ };
+ inputRecorder.start(250);
+ appendChatLine('录音', `开始录制 AI 输入：${describeTrack(inputRecordingTrack)}`, true);
+ updateRecordButton();
+ }
+
+ function stopInputRecording() {
+ if (inputRecorder?.state === 'recording') inputRecorder.stop();
+ }
+
+ function toggleInputRecording() {
+ if (inputRecorder?.state === 'recording') stopInputRecording();
+ else startInputRecording();
+ }
+
+ async function startVideoRecording() {
+ if (videoRecorder?.state === 'recording') return;
+ chooseRemoteVideoTrack('record-video');
+ if (!remoteUserVideoLocked || !isUsableTrack(remoteUserVideoTrack)) {
+ appendChatLine('录制视频', '没有可录制的上半部用户视频轨', true);
+ return;
+ }
+ if (!window.MediaRecorder) {
+ appendChatLine('录制视频', '当前浏览器不支持 MediaRecorder', true);
+ return;
+ }
+
+ videoRecordingChunks = [];
+ videoRecordingTrack = remoteUserVideoTrack;
+ videoRecordingStartedAt = Date.now();
+ const stream = new MediaStream([videoRecordingTrack]);
+ const mimeType = videoRecorderMimeType();
+ videoRecorder = new MediaRecorder(stream, mimeType ? { mimeType } : undefined);
+ videoRecorder.ondataavailable = (event) => {
+ if (event.data?.size > 0) videoRecordingChunks.push(event.data);
+ };
+ videoRecorder.onstop = () => {
+ const blob = new Blob(videoRecordingChunks, { type: videoRecorder?.mimeType || 'video/webm' });
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement('a');
+ link.href = url;
+ link.download = videoRecordingFileName(videoRecordingTrack);
+ document.body.appendChild(link);
+ link.click();
+ link.remove();
+ setTimeout(() => URL.revokeObjectURL(url), 30000);
+
+ const seconds = ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1);
+ appendChatLine('录制视频', `已保存 ${link.download}（${seconds}s，${(blob.size / 1024).toFixed(1)} KB）`, true);
+ videoRecorder = null;
+ videoRecordingChunks = [];
+ videoRecordingTrack = null;
+ updateVideoRecordButton();
+ };
+ videoRecorder.start(250);
+ appendChatLine('录制视频', `开始录制 AI 视频：${describeTrack(videoRecordingTrack)}`, true);
+ updateVideoRecordButton();
+ }
+
+ function stopVideoRecording() {
+ if (videoRecorder?.state === 'recording') videoRecorder.stop();
+ }
+
+ function toggleVideoRecording() {
+ if (videoRecorder?.state === 'recording') stopVideoRecording();
+ else startVideoRecording();
+ }
+
+ function generatePcId() {
+ const bytes = new Uint8Array(16);
+ crypto.getRandomValues(bytes);
+ return 'PC-' + Array.from(bytes).map((b) => b.toString(16).padStart(2, '0')).join('');
+ }
+
+ function isInternalPc(pc) {
+ return Boolean(pc?.__aiNativeCopilotInternal);
+ }
+
+ function isUsableTrack(track) {
+ return track && track.readyState === 'live' && !ignoredTrackIds.has(track.id);
+ }
+
+ function isLiveTrack(track) {
+ return track && track.readyState === 'live';
+ }
+
+ function candidateFor(track, candidates) {
+ let candidate = candidates.get(track.id);
+ if (!candidate) {
+ candidate = {
+ track,
+ receiver: null,
+ reasons: new Set(),
+ elements: new Set(),
+ firstSeenAt: Date.now(),
+ lastSeenAt: 0,
+ lastPacketAt: 0,
+ bytesReceived: 0,
+ packetsReceived: 0,
+ bytesDelta: 0,
+ packetsDelta: 0,
+ level: 0,
+ score: 0,
+ logged: false,
+ };
+ candidates.set(track.id, candidate);
+ }
+ candidate.track = track;
+ candidate.lastSeenAt = Date.now();
+ return candidate;
+ }
+
+ function businessSenderCandidateFor(sender, track, reason) {
+ if (!sender) return null;
+ const senderTrack = track || sender.track;
+ if (senderTrack && senderTrack.kind !== 'audio') return null;
+
+ const id = sender.__aiNativeCopilotBusinessSenderId || `business-sender-${businessSenderCandidates.size + 1}`;
+ sender.__aiNativeCopilotBusinessSenderId = id;
+
+ let candidate = businessSenderCandidates.get(id);
+ if (!candidate) {
+ candidate = {
+ id,
+ sender,
+ track: senderTrack || null,
+ originalTrack: senderTrack || null,
+ sources: new Set(),
+ bytesSent: 0,
+ packetsSent: 0,
+ bytesDelta: 0,
+ packetsDelta: 0,
+ score: 0,
+ logged: false,
+ };
+ businessSenderCandidates.set(id, candidate);
+ }
+
+ candidate.sender = sender;
+ candidate.track = senderTrack || sender.track || candidate.track;
+ if (!isAiTakeover && senderTrack && senderTrack !== aiResponseAudioTrack) {
+ candidate.originalTrack = senderTrack;
+ }
+ candidate.sources.add(reason);
+ wrapBusinessSender(sender);
+ return candidate;
+ }
+
+ function findReceiverForTrack(track) {
+ if (!track) return null;
+ return Array.from(window.__aiNativeCopilotReceivers || [])
+ .find((receiver) => receiver.track === track) || null;
+ }
+
+ function rememberReceiver(receiver, reason) {
+ if (!receiver?.track || isInternalPc(receiver.transport)) return;
+ window.__aiNativeCopilotReceivers.add(receiver);
+ const track = receiver.track;
+ if (!isUsableTrack(track)) return;
+
+ if (track.kind === 'audio') {
+ const candidate = candidateFor(track, audioCandidates);
+ candidate.receiver = receiver;
+ candidate.reasons.add(reason);
+ } else if (track.kind === 'video') {
+ const candidate = candidateFor(track, videoCandidates);
+ candidate.receiver = receiver;
+ candidate.reasons.add(reason);
+ chooseRemoteVideoTrack(reason);
+ }
+ }
+
+ function setVuWidth(id, level) {
+ const el = document.getElementById(id);
+ if (!el) return;
+ el.style.width = `${Math.min(100, level * 2500)}%`;
+ }
+
+ function renderVu() {
+ inputVuLevel *= 0.88;
+ aiOutVuLevel *= 0.88;
+ setVuWidth('native-copilot-input-vu', inputVuLevel);
+ setVuWidth('native-copilot-ai-vu', aiOutVuLevel);
+ requestAnimationFrame(renderVu);
+ }
+
+ function levelFromStats(stat, memory) {
+ let level = typeof stat.audioLevel === 'number' ? stat.audioLevel : 0;
+ if (typeof stat.totalAudioEnergy === 'number' && typeof stat.totalSamplesDuration === 'number') {
+ const prev = memory.get(stat.id);
+ memory.set(stat.id, {
+ energy: stat.totalAudioEnergy,
+ duration: stat.totalSamplesDuration,
+ });
+ if (prev) {
+ const energyDelta = stat.totalAudioEnergy - prev.energy;
+ const durationDelta = stat.totalSamplesDuration - prev.duration;
+ if (energyDelta > 0 && durationDelta > 0) {
+ level = Math.max(level, Math.sqrt(energyDelta / durationDelta));
+ }
+ }
+ }
+ return level;
+ }
+
+ async function refreshVuFromStats() {
+ try {
+ if (!(await checkLockedUserTracks('stats-before'))) return;
+ await refreshAudioCandidateStats();
+ chooseRemoteAudioTrack('stats');
+
+ await refreshBusinessSenderStats();
+ chooseBusinessAudioSender('stats');
+ await checkLockedUserTracks('stats-after');
+ } catch (error) {
+ // Stats APIs are best-effort across browsers.
+ }
+ }
+
+ async function refreshBusinessSenderStats() {
+ for (const candidate of businessSenderCandidates.values()) {
+ const sender = candidate.sender;
+ if (!sender?.getStats) continue;
+ candidate.track = sender.track || candidate.track;
+
+ const report = await sender.getStats();
+ report.forEach((stat) => {
+ const kind = stat.kind || stat.mediaType;
+ if (stat.type !== 'outbound-rtp' || kind !== 'audio') return;
+
+ const key = `${candidate.id}:${stat.id}`;
+ const prev = outboundStatsMemory.get(key);
+ const bytes = typeof stat.bytesSent === 'number' ? stat.bytesSent : 0;
+ const packets = typeof stat.packetsSent === 'number' ? stat.packetsSent : 0;
+ candidate.bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
+ candidate.packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
+ candidate.bytesSent = bytes;
+ candidate.packetsSent = packets;
+ outboundStatsMemory.set(key, { bytes, packets });
+
+ if (candidate.sender === businessAudioSender) {
+ businessOutboundStats.bytesDelta = candidate.bytesDelta;
+ businessOutboundStats.packetsDelta = candidate.packetsDelta;
+ businessOutboundStats.bytesSent = candidate.bytesSent;
+ businessOutboundStats.packetsSent = candidate.packetsSent;
+ aiOutVuLevel = Math.max(
+ aiOutVuLevel,
+ levelFromStats(stat, senderStatsMemory),
+ outputVuLevelFromStats()
+ );
+ }
+ });
+ }
+ }
+
+ async function refreshAudioCandidateStats() {
+ for (const candidate of audioCandidates.values()) {
+ if (!isUsableTrack(candidate.track)) continue;
+ const receiver = candidate.receiver || findReceiverForTrack(candidate.track);
+ if (!receiver?.getStats) continue;
+ candidate.receiver = receiver;
+
+ const report = await receiver.getStats();
+ report.forEach((stat) => {
+ const kind = stat.kind || stat.mediaType;
+ if (stat.type !== 'inbound-rtp' || kind !== 'audio') return;
+
+ const prev = inboundStatsMemory.get(stat.id);
+ const bytes = typeof stat.bytesReceived === 'number' ? stat.bytesReceived : 0;
+ const packets = typeof stat.packetsReceived === 'number' ? stat.packetsReceived : 0;
+ const bytesDelta = prev ? Math.max(0, bytes - prev.bytes) : 0;
+ const packetsDelta = prev ? Math.max(0, packets - prev.packets) : 0;
+ inboundStatsMemory.set(stat.id, { bytes, packets });
+
+ candidate.bytesReceived = bytes;
+ candidate.packetsReceived = packets;
+ candidate.bytesDelta = bytesDelta;
+ candidate.packetsDelta = packetsDelta;
+ candidate.level = levelFromStats(stat, receiverStatsMemory);
+ if (bytesDelta > 0 || packetsDelta > 0 || candidate.level > 0) {
+ candidate.lastPacketAt = Date.now();
+ }
+
+ if (candidate.track === remoteUserAudioTrack) {
+ inputVuLevel = Math.max(inputVuLevel, inputVuLevelFromCandidate(candidate));
+ }
+ });
+ }
+ }
+
+ async function outboundAudioBytes(sender) {
+ if (!sender?.getStats) return null;
+ try {
+ let total = 0;
+ const report = await sender.getStats();
+ report.forEach((stat) => {
+ const kind = stat.kind || stat.mediaType;
+ if (stat.type === 'outbound-rtp' && kind === 'audio' && typeof stat.bytesSent === 'number') {
+ total += stat.bytesSent;
+ }
+ });
+ return total;
+ } catch {
+ return null;
+ }
+ }
+
+ function aiReadyForMedia() {
+ if (!aiPc || !aiPc.remoteDescription) return false;
+ return (
+ aiPc.connectionState === 'connected'
+ || aiPc.iceConnectionState === 'connected'
+ || aiPc.iceConnectionState === 'completed'
+ );
+ }
+
+ function aiStateLabel() {
+ if (!aiPc) return 'pc=null';
+ return `conn=${aiPc.connectionState},ice=${aiPc.iceConnectionState},signaling=${aiPc.signalingState},remote=${aiPc.remoteDescription?.type || 'none'}`;
+ }
+
+ function waitForAiMediaReady(timeoutMs = 10000) {
+ if (aiReadyForMedia()) return Promise.resolve(true);
+ return new Promise((resolve) => {
+ const startedAt = Date.now();
+ const timer = setInterval(() => {
+ if (aiReadyForMedia()) {
+ clearInterval(timer);
+ resolve(true);
+ } else if (Date.now() - startedAt >= timeoutMs) {
+ clearInterval(timer);
+ resolve(false);
+ }
+ }, 250);
+ });
+ }
+
+ function scheduleAiInputHealthCheck(track) {
+ if (aiInputHealthTimer) clearTimeout(aiInputHealthTimer);
+ aiInputHealthTimer = setTimeout(async () => {
+ if (!aiAudioSender || aiAudioSender.track !== track) return;
+ log(`checking native AI input after media connects: ${aiStateLabel()}`);
+ const ready = await waitForAiMediaReady(10000);
+ if (!ready || !aiAudioSender || aiAudioSender.track !== track) {
+ log(`native AI input could not be verified: ${aiStateLabel()}`);
+ return;
+ }
+
+ const before = await outboundAudioBytes(aiAudioSender);
+ await new Promise((resolve) => setTimeout(resolve, 2500));
+ if (!aiAudioSender || aiAudioSender.track !== track) return;
+ const after = await outboundAudioBytes(aiAudioSender);
+ log(`native AI input packet check: before=${before}, after=${after}, ${aiStateLabel()}`, true);
+ }, 500);
+ }
+
+ function describeTrack(track) {
+ if (!track) return 'none';
+ return `${track.id}${track.muted ? ' muted' : ''}`;
+ }
+
+ function elementLayoutInfo(element) {
+ if (!element?.getBoundingClientRect || !document.documentElement.contains(element)) return null;
+ const rect = element.getBoundingClientRect();
+ const viewportHeight = window.innerHeight || document.documentElement.clientHeight || 1;
+ const centerY = rect.top + rect.height / 2;
+ const visibleWidth = Math.max(0, Math.min(rect.right, window.innerWidth || document.documentElement.clientWidth || rect.right) - Math.max(rect.left, 0));
+ const visibleHeight = Math.max(0, Math.min(rect.bottom, viewportHeight) - Math.max(rect.top, 0));
+ return {
+ tag: element.tagName,
+ top: Math.round(rect.top),
+ left: Math.round(rect.left),
+ width: Math.round(rect.width),
+ height: Math.round(rect.height),
+ centerY: Math.round(centerY),
+ upperHalf: centerY <= viewportHeight * 0.55,
+ visibleArea: Math.round(visibleWidth * visibleHeight),
+ paused: Boolean(element.paused),
+ muted: Boolean(element.muted),
+ volume: typeof element.volume === 'number' ? element.volume : null,
+ };
+ }
+
+ function candidateLayoutScore(candidate) {
+ let score = 0;
+ for (const element of candidate?.elements || []) {
+ const info = elementLayoutInfo(element);
+ if (!info) continue;
+ if (info.upperHalf) score += 9000;
+ else score -= 2500;
+ if (info.visibleArea > 0) score += Math.min(2500, info.visibleArea / 80);
+ if (info.tag === 'VIDEO') score += 1200;
+ if (info.tag === 'AUDIO') score += 300;
+ if (!info.paused) score += 700;
+ if (!info.muted && (info.volume ?? 1) > 0) score += 500;
+ }
+ return score;
+ }
+
+ function hasUpperHalfElement(candidate) {
+ return Array.from(candidate?.elements || []).some((element) => elementLayoutInfo(element)?.upperHalf);
+ }
+
+ function lockedTrackUsable(track, candidates) {
+ const candidate = candidates.get(track?.id);
+ return Boolean(candidate && isUsableTrack(candidate.track) && !candidate.track.muted);
+ }
+
+ async function handleLockedUserTrackLost(kind, reason) {
+ const message = `${kind === 'audio' ? '远端音频' : '远端视频'}轨道失效（${reason}），已断开 AI`;
+ appendChatLine('选轨', message, true);
+ log(message);
+ if (kind === 'audio') {
+ remoteUserAudioLocked = false;
+ remoteUserAudioTrack = null;
+ } else {
+ remoteUserVideoLocked = false;
+ remoteUserVideoTrack = null;
+ }
+ if (isAiTakeover) {
+ isAiTakeover = false;
+ updateModeButton();
+ await restoreBusinessAudio();
+ disconnectAi();
+ }
+ }
+
+ async function checkLockedUserTracks(reason) {
+ if (remoteUserAudioLocked && !lockedTrackUsable(remoteUserAudioTrack, audioCandidates)) {
+ await handleLockedUserTrackLost('audio', reason);
+ return false;
+ }
+ if (remoteUserVideoLocked && !lockedTrackUsable(remoteUserVideoTrack, videoCandidates)) {
+ await handleLockedUserTrackLost('video', reason);
+ return false;
+ }
+ return true;
+ }
+
+ function scoreAudioCandidate(candidate) {
+ if (!candidate || !isUsableTrack(candidate.track)) return -1;
+ const hasRecentPackets = Date.now() - candidate.lastPacketAt < 3000;
+ const hasAudibleLevel = candidate.level >= 0.0015;
+ let score = 0;
+ if (candidate.track.muted) score -= 5000;
+ else score += 800;
+ score += candidateLayoutScore(candidate);
+ if (candidate.receiver) score += 600;
+ if (candidate.bytesReceived > 0) score += 100;
+ if (candidate.bytesDelta > 0) score += 500 + Math.min(900, candidate.bytesDelta * 2);
+ if (candidate.packetsDelta > 0) score += 300 + Math.min(500, candidate.packetsDelta * 8);
+ if (hasRecentPackets) score += 400;
+ if (hasAudibleLevel) score += 5000;
+ score += Math.min(12000, candidate.level * 900000);
+ if (!hasAudibleLevel && hasRecentPackets) score -= 1800;
+ if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 80;
+ return score;
+ }
+
+ function scoreVideoCandidate(candidate) {
+ if (!candidate || !isUsableTrack(candidate.track)) return -1;
+ let score = 0;
+ if (candidate.track.muted) score -= 1000;
+ else score += 500;
+ if (candidate.receiver) score += 600;
+ score += candidateLayoutScore(candidate);
+ for (const element of candidate.elements || []) {
+ const info = elementLayoutInfo(element);
+ if (!info) continue;
+ if (info.tag === 'VIDEO') score += 2500;
+ if (info.visibleArea > 0) score += Math.min(4000, info.visibleArea / 50);
+ }
+ if (candidate.reasons.has('media-srcObject') || candidate.reasons.has('media-scan')) score += 100;
+ return score;
+ }
+
+ function inputVuLevelFromCandidate(candidate) {
+ if (!candidate) return 0;
+ return Math.max(
+ candidate.level,
+ Math.min(0.04, candidate.bytesDelta / 16000),
+ Math.min(0.02, candidate.packetsDelta / 800)
+ );
+ }
+
+ function updateSelectedInputLabel() {
+ const label = document.getElementById('native-copilot-selected-input');
+ if (!label) return;
+ const candidate = audioCandidates.get(remoteUserAudioTrack?.id);
+ if (!candidate) {
+ label.textContent = '等待远端音轨';
+ return;
+ }
+ const layout = Array.from(candidate.elements || []).map(elementLayoutInfo).find(Boolean);
+ const where = layout ? (layout.upperHalf ? 'upper' : 'lower') : 'no-layout';
+ label.textContent = `${remoteUserAudioTrack.id.slice(0, 8)} ${where} level=${candidate.level.toFixed(4)} score=${Math.round(candidate.score || scoreAudioCandidate(candidate))}`;
+ }
+
+ function outputVuLevelFromStats() {
+ return Math.max(
+ Math.min(0.12, businessOutboundStats.bytesDelta / 12000),
+ Math.min(0.04, businessOutboundStats.packetsDelta / 500)
+ );
+ }
+
+ function isKnownRemoteAudioTrack(track) {
+ return Boolean(track && (
+ track === remoteUserAudioTrack
+ || audioCandidates.has(track.id)
+ ));
+ }
+
+ function isAiOutputTrack(track) {
+ return Boolean(track && track === aiResponseAudioTrack);
+ }
+
+ function scoreBusinessSenderCandidate(candidate) {
+ if (!candidate?.sender) return -1;
+ const track = candidate.sender.track || candidate.track;
+ let score = 0;
+
+ if (track?.kind === 'audio') score += 500;
+ else if (!track) score -= 1000;
+
+ if (track?.readyState === 'live') score += 700;
+ if (track && !track.muted) score += 300;
+ if (candidate.sources.has('pc-addTrack')) score += 5000;
+ if (candidate.sources.has('pc-addTransceiver')) score -= 1000;
+ if (track && localCaptureTrackIds.has(track.id)) score += 2500;
+ if (candidate.originalTrack && localCaptureTrackIds.has(candidate.originalTrack.id)) score += 2500;
+ if (isKnownRemoteAudioTrack(track)) score -= 9000;
+ if (candidate.originalTrack && isKnownRemoteAudioTrack(candidate.originalTrack)) score -= 9000;
+ if (isAiOutputTrack(track)) score += 4500;
+ if (candidate.bytesSent > 0) score += 600;
+ if (candidate.bytesDelta > 0) score += 1600 + Math.min(1800, candidate.bytesDelta * 4);
+ if (candidate.packetsDelta > 0) score += 800 + Math.min(900, candidate.packetsDelta * 10);
+
+ return score;
+ }
+
+ function chooseBusinessAudioSender(reason) {
+ let best = null;
+ let bestScore = -1;
+ for (const candidate of businessSenderCandidates.values()) {
+ candidate.score = scoreBusinessSenderCandidate(candidate);
+ if (candidate.score > bestScore) {
+ best = candidate;
+ bestScore = candidate.score;
+ }
+ }
+
+ if (!best || bestScore < 0) return false;
+ if (businessAudioSender === best.sender) return true;
+
+ businessAudioSender = best.sender;
+ businessOriginalAudioTrack = best.originalTrack || best.sender.track || businessOriginalAudioTrack;
+ log(`selected business audio sender (${reason}): ${best.id}, track=${describeTrack(best.sender.track)}, score=${Math.round(bestScore)}, sources=${Array.from(best.sources).join('+')}`, true);
+ attachAiOutputToBusiness();
+ return true;
+ }
+
+ function chooseRemoteAudioTrack(reason) {
+ const lockedAudio = audioCandidates.get(remoteUserAudioTrack?.id);
+ if (remoteUserAudioLocked && lockedAudio && isUsableTrack(lockedAudio.track) && !lockedAudio.track.muted) {
+ updateSelectedInputLabel();
+ return true;
+ }
+ if (remoteUserAudioLocked) return false;
+
+ let best = null;
+ let bestScore = -1;
+ for (const candidate of audioCandidates.values()) {
+ candidate.score = scoreAudioCandidate(candidate);
+ if (candidate.score > bestScore) {
+ best = candidate;
+ bestScore = candidate.score;
+ }
+ }
+
+ if (!best || bestScore < 0) return false;
+ if (remoteUserAudioTrack === best.track) {
+ updateSelectedInputLabel();
+ return true;
+ }
+
+ const current = audioCandidates.get(remoteUserAudioTrack?.id);
+ const currentScore = scoreAudioCandidate(current);
+ const currentHealthy = current && isUsableTrack(current.track) && !current.track.muted
+ && (current.level >= 0.0015 || hasUpperHalfElement(current))
+ && Date.now() - current.lastPacketAt < 3000;
+ if (currentHealthy && bestScore < currentScore + 1800) {
+ updateSelectedInputLabel();
+ return true;
+ }
+
+ remoteUserAudioTrack = best.track;
+ remoteUserAudioLocked = hasUpperHalfElement(best);
+ log(`${remoteUserAudioLocked ? 'locked' : 'selected'} remote audio track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, level=${best.level.toFixed(4)}, bytesDelta=${best.bytesDelta}, packetsDelta=${best.packetsDelta}`, true);
+ updateSelectedInputLabel();
+ attachRemoteInputToAi();
+ return true;
+ }
+
+ function chooseRemoteVideoTrack(reason) {
+ const lockedVideo = videoCandidates.get(remoteUserVideoTrack?.id);
+ if (remoteUserVideoLocked && lockedVideo && isUsableTrack(lockedVideo.track) && !lockedVideo.track.muted) {
+ return true;
+ }
+ if (remoteUserVideoLocked) return false;
+
+ let best = null;
+ let bestScore = -1;
+ for (const candidate of videoCandidates.values()) {
+ candidate.score = scoreVideoCandidate(candidate);
+ if (candidate.score > bestScore) {
+ best = candidate;
+ bestScore = candidate.score;
+ }
+ }
+
+ if (!best || bestScore < 0) return false;
+ if (remoteUserVideoTrack === best.track) return true;
+
+ const current = videoCandidates.get(remoteUserVideoTrack?.id);
+ const currentScore = scoreVideoCandidate(current);
+ if (hasUpperHalfElement(current) && bestScore < currentScore + 1800) return true;
+
+ remoteUserVideoTrack = best.track;
+ remoteUserVideoLocked = hasUpperHalfElement(best);
+ log(`${remoteUserVideoLocked ? 'locked' : 'selected'} remote video track (${reason}): ${describeTrack(best.track)}, score=${Math.round(bestScore)}, layout=${Math.round(candidateLayoutScore(best))}`, true);
+ attachRemoteVideoToAi();
+ return true;
+ }
+
+ function ensureUserTracksBeforeConnect() {
+ chooseRemoteAudioTrack('pre-connect');
+ chooseRemoteVideoTrack('pre-connect');
+
+ if (shouldSendAudioToAi() && (!remoteUserAudioLocked || !lockedTrackUsable(remoteUserAudioTrack, audioCandidates))) {
+ appendChatLine('选轨', '未锁定上半部用户音频轨，暂不连接 AI', true);
+ return false;
+ }
+ if (shouldSendVideoToAi() && (!remoteUserVideoLocked || !lockedTrackUsable(remoteUserVideoTrack, videoCandidates))) {
+ appendChatLine('选轨', '未锁定上半部用户视频轨，暂不连接 AI', true);
+ return false;
+ }
+ return true;
+ }
+
+ function rememberRemoteTrack(track, reason, element = null) {
+ if (!isUsableTrack(track)) return;
+
+ if (track.kind === 'audio') {
+ const candidate = candidateFor(track, audioCandidates);
+ candidate.receiver = candidate.receiver || findReceiverForTrack(track);
+ candidate.reasons.add(reason);
+ if (element) candidate.elements.add(element);
+ if (!candidate.logged) {
+ candidate.logged = true;
+ log(`captured remote audio candidate (${reason}): ${describeTrack(track)}`, true);
+ }
+ chooseRemoteAudioTrack(reason);
+ return;
+ }
+
+ if (track.kind === 'video') {
+ const candidate = candidateFor(track, videoCandidates);
+ candidate.receiver = candidate.receiver || findReceiverForTrack(track);
+ candidate.reasons.add(reason);
+ if (element) candidate.elements.add(element);
+ if (!candidate.logged) {
+ candidate.logged = true;
+ log(`captured remote video candidate (${reason}): ${describeTrack(track)}`, true);
+ }
+ chooseRemoteVideoTrack(reason);
+ }
+ }
+
+ function rememberRemoteStream(stream, reason, element = null) {
+ if (!stream || !(stream instanceof MediaStream)) return;
+ if (stream.__aiNativeCopilotInternal) return;
+
+ const liveTracks = stream.getTracks().filter(isUsableTrack);
+ if (liveTracks.length === 0) return;
+
+ remoteStreams.add(stream);
+ for (const track of liveTracks) rememberRemoteTrack(track, reason, element);
+ chooseRemoteVideoTrack(reason);
+ }
+
+ function rememberBusinessAudioSender(sender, track, reason) {
+ const candidate = businessSenderCandidateFor(sender, track, reason);
+ if (!candidate) return;
+ if (!candidate.logged) {
+ candidate.logged = true;
+ log(`captured business audio sender candidate (${reason}): ${candidate.id}, track=${describeTrack(candidate.track)}`, true);
+ }
+ chooseBusinessAudioSender(reason);
+ }
+
+ function wrapBusinessSender(sender) {
+ if (!sender || sender.__aiNativeCopilotWrapped) return;
+ const originalReplaceTrack = sender.replaceTrack?.bind(sender);
+ if (!originalReplaceTrack) return;
+
+ sender.replaceTrack = async function (track) {
+ const result = await originalReplaceTrack(track);
+ const candidate = businessSenderCandidateFor(sender, track, 'replaceTrack');
+ if (candidate) {
+ candidate.track = track || sender.track || candidate.track;
+ }
+ if (!isAiTakeover && track?.kind === 'audio' && track !== aiResponseAudioTrack) {
+ if (candidate) candidate.originalTrack = track;
+ chooseBusinessAudioSender('replaceTrack');
+ }
+ return result;
+ };
+ sender.__aiNativeCopilotWrapped = true;
+ }
+
+ async function attachRemoteInputToAi() {
+ if (!shouldSendAudioToAi()) return false;
+ if (!aiAudioSender || !isUsableTrack(remoteUserAudioTrack)) return false;
+ if (aiAudioSender.track === remoteUserAudioTrack) return true;
+
+ try {
+ await aiAudioSender.replaceTrack(remoteUserAudioTrack);
+ log(`AI input uses native remote audio track (${remoteUserAudioTrack.id})`, true);
+ scheduleAiInputHealthCheck(remoteUserAudioTrack);
+ return true;
+ } catch (error) {
+ log(`failed to attach native remote audio to AI: ${error.message || error}`);
+ return false;
+ }
+ }
+
+ async function attachRemoteVideoToAi() {
+ if (!shouldSendVideoToAi()) return false;
+ if (!aiVideoSender || !isUsableTrack(remoteUserVideoTrack)) return false;
+ if (aiVideoSender.track === remoteUserVideoTrack) return true;
+
+ try {
+ await aiVideoSender.replaceTrack(remoteUserVideoTrack);
+ log('AI video uses native remote video track', true);
+ return true;
+ } catch (error) {
+ log(`failed to attach native remote video to AI: ${error.message || error}`);
+ return false;
+ }
+ }
+
+ async function attachAiOutputToBusiness() {
+ chooseBusinessAudioSender('before-ai-output');
+ if (!isAiTakeover || !businessAudioSender || !isLiveTrack(aiResponseAudioTrack)) {
+ log(`AI output waiting: takeover=${isAiTakeover}, businessSender=${Boolean(businessAudioSender)}, aiTrack=${describeTrack(aiResponseAudioTrack)}`);
+ return false;
+ }
+ if (businessAudioSender.track === aiResponseAudioTrack) return true;
+
+ try {
+ await businessAudioSender.replaceTrack(aiResponseAudioTrack);
+ aiTrackAttachedToBusiness = true;
+ log(`business call now uses native AI response audio track (${aiResponseAudioTrack.id})`, true);
+ scheduleAiOutputHealthCheck(aiResponseAudioTrack);
+ return true;
+ } catch (error) {
+ aiTrackAttachedToBusiness = false;
+ log(`failed to attach AI audio to business call: ${error.message || error}`);
+ return false;
+ }
+ }
+
+ function scheduleAiOutputHealthCheck(track) {
+ if (aiOutputHealthTimer) clearTimeout(aiOutputHealthTimer);
+ aiOutputHealthTimer = setTimeout(async () => {
+ if (!businessAudioSender || businessAudioSender.track !== track) {
+ log(`AI output check skipped: senderTrack=${describeTrack(businessAudioSender?.track)}, aiTrack=${describeTrack(track)}`);
+ return;
+ }
+
+ const before = await outboundAudioBytes(businessAudioSender);
+ await new Promise((resolve) => setTimeout(resolve, 2500));
+ if (!businessAudioSender || businessAudioSender.track !== track) return;
+ const after = await outboundAudioBytes(businessAudioSender);
+ log(`AI output packet check: before=${before}, after=${after}, senderTrack=${describeTrack(businessAudioSender.track)}`, true);
+ }, 800);
+ }
+
+ async function restoreBusinessAudio() {
+ if (!businessAudioSender || !aiTrackAttachedToBusiness) return;
+
+ try {
+ await businessAudioSender.replaceTrack(businessOriginalAudioTrack || null);
+ log('business call restored original audio track', true);
+ } catch (error) {
+ log(`failed to restore business audio track: ${error.message || error}`);
+ } finally {
+ aiTrackAttachedToBusiness = false;
+ }
+ }
+
+ function wrapTrackListener(listener) {
+ if (!listener || listener.__aiNativeCopilotWrapped) return listener;
+
+ const wrapped = function (event) {
+ if (!isInternalPc(this) && event?.track) {
+ rememberReceiver(event.receiver, 'pc-track');
+ rememberRemoteTrack(event.track, 'pc-track');
+ for (const stream of event.streams || []) rememberRemoteStream(stream, 'pc-track-stream');
+ }
+
+ if (typeof listener === 'function') return listener.apply(this, arguments);
+ if (listener?.handleEvent) return listener.handleEvent(event);
+ return undefined;
+ };
+
+ wrapped.__aiNativeCopilotWrapped = true;
+ return wrapped;
+ }
+
+ RTCPeerConnection.prototype.addEventListener = function (type, listener, options) {
+ if (type === 'track') {
+ return originalAddEventListener.call(this, type, wrapTrackListener(listener), options);
+ }
+ return originalAddEventListener.call(this, type, listener, options);
+ };
+
+ try {
+ const descriptor = Object.getOwnPropertyDescriptor(RTCPeerConnection.prototype, 'ontrack');
+ Object.defineProperty(RTCPeerConnection.prototype, 'ontrack', {
+ configurable: true,
+ enumerable: true,
+ get() {
+ return descriptor?.get ? descriptor.get.call(this) : this.__aiNativeCopilotOnTrack || null;
+ },
+ set(listener) {
+ const wrapped = wrapTrackListener(listener);
+ this.__aiNativeCopilotOnTrack = wrapped;
+ if (descriptor?.set) descriptor.set.call(this, wrapped);
+ else this.addEventListener('track', wrapped);
+ },
+ });
+ } catch (error) {
+ console.warn('[AI Native Copilot] failed to wrap ontrack', error);
+ }
+
+ RTCPeerConnection.prototype.addTrack = function (track, ...streams) {
+ const sender = originalAddTrack.call(this, track, ...streams);
+ if (!isInternalPc(this) && track?.kind === 'audio') {
+ rememberBusinessAudioSender(sender, track, 'pc-addTrack');
+ }
+ return sender;
+ };
+
+ if (originalAddTransceiver) {
+ RTCPeerConnection.prototype.addTransceiver = function (trackOrKind, init) {
+ const transceiver = originalAddTransceiver.call(this, trackOrKind, init);
+ if (!isInternalPc(this)) {
+ const kind = trackOrKind instanceof MediaStreamTrack ? trackOrKind.kind : trackOrKind;
+ if (kind === 'audio') {
+ const track = trackOrKind instanceof MediaStreamTrack ? trackOrKind : transceiver.sender?.track;
+ rememberBusinessAudioSender(transceiver.sender, track, 'pc-addTransceiver');
+ }
+ }
+ return transceiver;
+ };
+ }
+
+ RTCPeerConnection.prototype.setRemoteDescription = function () {
+ const result = originalSetRemoteDescription.apply(this, arguments);
+ if (!isInternalPc(this)) {
+ Promise.resolve(result).then(() => {
+ try {
+ for (const receiver of this.getReceivers?.() || []) {
+ rememberReceiver(receiver, 'receiver-after-srd');
+ rememberRemoteTrack(receiver.track, 'receiver-after-srd');
+ }
+ for (const sender of this.getSenders?.() || []) {
+ rememberBusinessAudioSender(sender, sender.track, 'sender-after-srd');
+ }
+ for (const stream of this.getRemoteStreams?.() || []) {
+ rememberRemoteStream(stream, 'remote-stream-after-srd');
+ }
+ } catch (error) {
+ console.warn('[AI Native Copilot] failed to scan receivers', error);
+ }
+ });
+ }
+ return result;
+ };
+
+ if (mediaSrcObjectDescriptor?.set) {
+ Object.defineProperty(HTMLMediaElement.prototype, 'srcObject', {
+ configurable: true,
+ enumerable: mediaSrcObjectDescriptor.enumerable,
+ get() {
+ return mediaSrcObjectDescriptor.get.call(this);
+ },
+ set(stream) {
+ if ((this.tagName === 'AUDIO' || this.tagName === 'VIDEO') && !this.__aiNativeCopilotElement) {
+ rememberRemoteStream(stream, 'media-srcObject', this);
+ }
+ return mediaSrcObjectDescriptor.set.call(this, stream);
+ },
+ });
+ }
+
+ if (originalGetUserMedia) {
+ navigator.mediaDevices.getUserMedia = async function (constraints) {
+ const stream = await originalGetUserMedia(constraints);
+ for (const track of stream.getAudioTracks()) {
+ localCaptureTrackIds.add(track.id);
+ }
+ chooseBusinessAudioSender('getUserMedia');
+ renderPanelSoon();
+ return stream;
+ };
+ }
+
+ setInterval(async () => {
+ if (!(await checkLockedUserTracks('scan'))) return;
+ for (const element of document.querySelectorAll('audio, video')) {
+ if (element.__aiNativeCopilotElement) continue;
+ rememberRemoteStream(element.srcObject, 'media-scan', element);
+ }
+
+ for (const stream of Array.from(remoteStreams)) {
+ rememberRemoteStream(stream, 'stream-rescan');
+ }
+ await checkLockedUserTracks('scan-after');
+ }, 1000);
+
+ window.__aiNativeCopilotDump = function () {
+ const audio = Array.from(audioCandidates.values()).map((candidate) => ({
+ id: candidate.track.id,
+ label: candidate.track.label,
+ selected: candidate.track === remoteUserAudioTrack,
+ enabled: candidate.track.enabled,
+ muted: candidate.track.muted,
+ readyState: candidate.track.readyState,
+ hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)),
+ bytesReceived: candidate.bytesReceived,
+ packetsReceived: candidate.packetsReceived,
+ bytesDelta: candidate.bytesDelta,
+ packetsDelta: candidate.packetsDelta,
+ level: candidate.level,
+ audible: candidate.level >= 0.0015,
+ layoutScore: Math.round(candidateLayoutScore(candidate)),
+ elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean),
+ vuLevel: inputVuLevelFromCandidate(candidate),
+ score: Math.round(scoreAudioCandidate(candidate)),
+ reasons: Array.from(candidate.reasons),
+ }));
+ const video = Array.from(videoCandidates.values()).map((candidate) => ({
+ id: candidate.track.id,
+ label: candidate.track.label,
+ selected: candidate.track === remoteUserVideoTrack,
+ muted: candidate.track.muted,
+ readyState: candidate.track.readyState,
+ hasReceiver: Boolean(candidate.receiver || findReceiverForTrack(candidate.track)),
+ layoutScore: Math.round(candidateLayoutScore(candidate)),
+ score: Math.round(scoreVideoCandidate(candidate)),
+ elements: Array.from(candidate.elements || []).map(elementLayoutInfo).filter(Boolean),
+ reasons: Array.from(candidate.reasons),
+ }));
+ const businessSenders = Array.from(businessSenderCandidates.values()).map((candidate) => ({
+ id: candidate.id,
+ selected: candidate.sender === businessAudioSender,
+ track: describeTrack(candidate.sender.track || candidate.track),
+ originalTrack: describeTrack(candidate.originalTrack),
+ isRemoteTrack: isKnownRemoteAudioTrack(candidate.sender.track || candidate.track),
+ isLocalCaptureTrack: Boolean(candidate.sender.track && localCaptureTrackIds.has(candidate.sender.track.id)),
+ bytesSent: candidate.bytesSent,
+ packetsSent: candidate.packetsSent,
+ bytesDelta: candidate.bytesDelta,
+ packetsDelta: candidate.packetsDelta,
+ score: Math.round(scoreBusinessSenderCandidate(candidate)),
+ sources: Array.from(candidate.sources),
+ }));
+ return {
+ selectedAudio: describeTrack(remoteUserAudioTrack),
+ selectedVideo: describeTrack(remoteUserVideoTrack),
+ audioLocked: remoteUserAudioLocked,
+ videoLocked: remoteUserVideoLocked,
+ aiInputTrack: describeTrack(aiAudioSender?.track),
+ aiOutputTrack: describeTrack(aiResponseAudioTrack),
+ businessSenderTrack: describeTrack(businessAudioSender?.track),
+ aiTrackAttachedToBusiness,
+ videoRecording: videoRecorder?.state === 'recording' ? {
+ track: describeTrack(videoRecordingTrack),
+ seconds: ((Date.now() - videoRecordingStartedAt) / 1000).toFixed(1),
+ chunks: videoRecordingChunks.length,
+ } : null,
+ businessOutboundStats: { ...businessOutboundStats },
+ aiState: aiStateLabel(),
+ audio,
+ video,
+ businessSenders,
+ };
+ };
+
+ async function connectAi(signalingUrl, assistantId) {
+ if (!ensureUserTracksBeforeConnect()) {
+ setStatus('disconnected');
+ return false;
+ }
+ disconnectAi();
+
+ const pcId = generatePcId();
+ aiPc = new RTCPeerConnection({ iceServers: [{ urls: 'stun:stun.l.google.com:19302' }] });
+ aiPc.__aiNativeCopilotInternal = true;
+ aiPc.onconnectionstatechange = () => log(`AI PC connectionState=${aiPc.connectionState}`);
+ aiPc.oniceconnectionstatechange = () => log(`AI PC iceConnectionState=${aiPc.iceConnectionState}`);
+
+ aiDc = aiPc.createDataChannel('chat');
+ aiDc.onopen = () => {
+ setStatus('connected');
+ log('AI data channel open', true);
+ aiDc.send(JSON.stringify({ type: 'client-ready' }));
+ };
+ aiDc.onclose = () => setStatus('disconnected');
+ aiDc.onmessage = (event) => {
+ try {
+ const msg = JSON.parse(event.data);
+ const content = msg.delta || msg.content;
+ if (msg.type === 'transcript') {
+ logUserTranscript(content);
+ } else if (msg.type === 'assistant-text-start') {
+ startAssistantLine();
+ } else if (msg.type === 'assistant-text-delta') {
+ appendAssistantDelta(content);
+ } else if (msg.type === 'assistant-text-end') {
+ finishAssistantLine();
+ } else {
+ log(`data channel message: ${msg.type}`);
+ }
+ } catch {
+ appendChatLine('DATA', event.data);
+ }
+ };
+
+ const sendAudio = shouldSendAudioToAi();
+ const audioTransceiver = sendAudio && isUsableTrack(remoteUserAudioTrack)
+ ? aiPc.addTransceiver(remoteUserAudioTrack, { direction: 'sendrecv' })
+ : aiPc.addTransceiver('audio', { direction: sendAudio ? 'sendrecv' : 'recvonly' });
+ aiAudioSender = audioTransceiver.sender;
+ log(!sendAudio
+ ? 'AI audio uplink disabled by panel setting'
+ : isUsableTrack(remoteUserAudioTrack)
+ ? 'AI input starts with native remote audio track'
+ : 'AI input waiting for native remote audio track', true);
+
+ const sendVideo = shouldSendVideoToAi();
+ if (sendVideo) {
+ const videoTransceiver = isUsableTrack(remoteUserVideoTrack)
+ ? aiPc.addTransceiver(remoteUserVideoTrack, { direction: 'sendonly' })
+ : aiPc.addTransceiver('video', { direction: 'sendonly' });
+ aiVideoSender = videoTransceiver.sender;
+ if (isUsableTrack(remoteUserVideoTrack)) log('AI video starts with native remote video track', true);
+ } else {
+ aiVideoSender = null;
+ log('AI video uplink disabled by panel setting');
+ }
+ if (sendAudio && isUsableTrack(remoteUserAudioTrack)) scheduleAiInputHealthCheck(remoteUserAudioTrack);
+
+ aiPc.ontrack = (event) => {
+ if (event.track.kind !== 'audio') return;
+ ignoredTrackIds.add(event.track.id);
+ aiResponseAudioTrack = event.track;
+ log('received native AI response audio track', true);
+ attachAiOutputToBusiness();
+ };
+
+ aiPc.onicecandidate = (event) => {
+ if (aiWs?.readyState !== WebSocket.OPEN) return;
+ aiWs.send(JSON.stringify({
+ type: 'ice-candidate',
+ payload: {
+ pc_id: pcId,
+ candidate: event.candidate ? {
+ candidate: event.candidate.candidate,
+ sdpMid: event.candidate.sdpMid,
+ sdpMLineIndex: event.candidate.sdpMLineIndex,
+ } : null,
+ },
+ }));
+ };
+
+ setStatus('connecting');
+ aiWs = new WebSocket(signalingUrl);
+ aiWs.onopen = async () => {
+ const offer = await aiPc.createOffer();
+ await aiPc.setLocalDescription(offer);
+ aiWs.send(JSON.stringify({
+ type: 'offer',
+ payload: {
+ pc_id: pcId,
+ sdp: aiPc.localDescription.sdp,
+ type: aiPc.localDescription.type,
+ assistant_id: assistantId,
+ vision_enabled: sendVideo,
+ },
+ }));
+ log('sent AI WebRTC offer', true);
+ };
+ aiWs.onmessage = async (event) => {
+ try {
+ const msg = JSON.parse(event.data);
+ if (msg.type === 'answer') {
+ await aiPc.setRemoteDescription({ type: 'answer', sdp: msg.payload.sdp });
+ log(`AI WebRTC answer applied; ${aiStateLabel()}`, true);
+ } else if (msg.type === 'ice-candidate' && msg.payload?.candidate) {
+ await aiPc.addIceCandidate(msg.payload.candidate).catch(() => {});
+ } else if (msg.type === 'error') {
+ throw new Error(msg.payload?.message || 'server error');
+ }
+ } catch (error) {
+ log(`signaling error: ${error.message || error}`);
+ }
+ };
+ aiWs.onerror = () => {
+ setStatus('failed');
+ log('AI signaling websocket error');
+ };
+ aiWs.onclose = () => setStatus('disconnected');
+
+ attachRemoteInputToAi();
+ if (sendVideo) attachRemoteVideoToAi();
+ return true;
+ }
+
+ function disconnectAi() {
+ if (aiDc) {
+ try { aiDc.close(); } catch {}
+ aiDc = null;
+ }
+ if (aiPc) {
+ try { aiPc.close(); } catch {}
+ aiPc = null;
+ }
+ if (aiWs) {
+ try { aiWs.close(); } catch {}
+ aiWs = null;
+ }
+ aiAudioSender = null;
+ aiVideoSender = null;
+ aiResponseAudioTrack = null;
+ if (inputRecorder?.state === 'recording') stopInputRecording();
+ if (videoRecorder?.state === 'recording') stopVideoRecording();
+ setStatus('disconnected');
+ }
+
+ function setStatus(status) {
+ const el = document.getElementById('native-copilot-status');
+ if (!el) return;
+ const labels = {
+ disconnected: '未连接',
+ connecting: '连接中',
+ connected: '已连接',
+ failed: '连接失败',
+ };
+ el.textContent = labels[status] || status;
+ }
+
+ function updateModeButton() {
+ const button = document.getElementById('native-copilot-toggle');
+ if (!button) return;
+ if (isAiTakeover) {
+ button.textContent = '当前：AI 接管｜点击切回人工';
+ button.style.background = '#8b5cf6';
+ } else {
+ button.textContent = '当前：人工接管｜点击切换 AI';
+ button.style.background = '#10b981';
+ }
+ }
+
+ function renderPanel() {
+ if (!document.body || document.getElementById('native-copilot-panel')) return;
+
+ const panel = document.createElement('div');
+ panel.id = 'native-copilot-panel';
+ panel.style.cssText = [
+ 'position:fixed',
+ 'top:20px',
+ 'left:20px',
+ 'width:360px',
+ 'z-index:999999',
+ 'box-sizing:border-box',
+ 'padding:16px',
+ 'border-radius:10px',
+ 'background:rgba(17,24,39,.96)',
+ 'color:white',
+ 'font-family:system-ui,-apple-system,BlinkMacSystemFont,sans-serif',
+ 'box-shadow:0 12px 32px rgba(0,0,0,.35)',
+ ].join(';');
+
+ panel.innerHTML = `
+
+ AI 原生选轨
+ 未连接
+
+
+ 音频输入：布局优先自动选轨
+ 视频输入：远端视频轨
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 等待交互...
+
+ `;
+
+ document.body.appendChild(panel);
+
+ const toggle = document.getElementById('native-copilot-toggle');
+ const recordButton = document.getElementById('native-copilot-record');
+ const videoRecordButton = document.getElementById('native-copilot-record-video');
+ const urlInput = document.getElementById('native-copilot-url');
+ const assistantInput = document.getElementById('native-copilot-assistant');
+ const sendAudioInput = document.getElementById('native-copilot-send-audio');
+ const sendVideoInput = document.getElementById('native-copilot-send-video');
+
+ sendAudioInput.checked = shouldSendAudioToAi();
+ sendVideoInput.checked = shouldSendVideoToAi();
+ sendAudioInput.addEventListener('change', () => {
+ writeBoolSetting(AUDIO_UPLINK_STORAGE_KEY, sendAudioInput.checked);
+ log(`AI audio uplink setting: ${sendAudioInput.checked ? 'on' : 'off'}`);
+ });
+ sendVideoInput.addEventListener('change', () => {
+ writeBoolSetting(VIDEO_UPLINK_STORAGE_KEY, sendVideoInput.checked);
+ log(`AI video uplink setting: ${sendVideoInput.checked ? 'on' : 'off'}`);
+ });
+ recordButton.addEventListener('click', toggleInputRecording);
+ videoRecordButton.addEventListener('click', toggleVideoRecording);
+ updateRecordButton();
+ updateVideoRecordButton();
+ updateSelectedInputLabel();
+ updateModeButton();
+
+ toggle.addEventListener('click', async () => {
+ if (!isAiTakeover && !assistantInput.value.trim()) {
+ alert('请先填写助手 ID。');
+ return;
+ }
+
+ if (!isAiTakeover && !ensureUserTracksBeforeConnect()) {
+ updateModeButton();
+ return;
+ }
+
+ isAiTakeover = !isAiTakeover;
+ updateModeButton();
+ if (isAiTakeover) {
+ const connected = await connectAi(urlInput.value.trim(), assistantInput.value.trim());
+ if (connected) await attachAiOutputToBusiness();
+ else {
+ isAiTakeover = false;
+ updateModeButton();
+ }
+ } else {
+ await restoreBusinessAudio();
+ disconnectAi();
+ }
+ });
+ }
+
+ function renderPanelSoon() {
+ try { renderPanel(); } catch (error) { console.warn('[AI Native Copilot] panel error', error); }
+ }
+
+ if (document.readyState === 'loading') {
+ document.addEventListener('DOMContentLoaded', renderPanelSoon, { once: true });
+ } else {
+ renderPanelSoon();
+ }
+ requestAnimationFrame(renderVu);
+ setInterval(refreshVuFromStats, 250);
+ setTimeout(renderPanelSoon, 1500);
+ setTimeout(renderPanelSoon, 3000);
+})();